俊瑶先森

version: '3'
services:
  mysql:
    restart: always
    image: mysql:5.7.23
    container_name: mysql
    ports:
      - 3306:3306
    volumes:
      - ./conf:/etc/mysql/conf.d
      - ./logs:/var/log/mysql
      - ./data:/var/lib/mysql

cannot read directory '/var/lib/mysql/': Permission denied

# allow the container to be started with `--user`
if [ "$1" = 'mysqld' -a -z "$wantHelp" -a "$(id -u)" = '0' ]; then
        _check_config "$@"
        DATADIR="$(_get_config 'datadir' "$@")"
        mkdir -p "$DATADIR"
        chown -R mysql:mysql "$DATADIR"
        exec gosu mysql "$BASH_SOURCE" "$@"
fi

// 容器内查看用户名为`mysql` 对应的用户ID和组ID为`999`
cat /etc/passwd | grep mysql

// 宿主机器查看999所对应的用户名
cat /etc/passwd | grep 999

[root@localhost ~]# getenforce
Enforcing
[root@localhost ~]# setenforce 0
[root@localhost ~]# getenforce
Permissive
[root@localhost ~]# systemctl restart docker
[root@localhost ~]# vi /etc/selinux/config
#修改 SELINUX=disabled
[root@localhost ~]# init 6
#重启系统  

FROM primetoninc/jdk:1.8
# RUN yum install net-tools -y
COPY fe /opt/fe
WORKDIR /opt/fe
EXPOSE 8030 9030
ENTRYPOINT ["/opt/fe/bin/start_fe.sh"]

docker build -t doris-fe:0.15.0 -f Dockerfile_fe .

FROM primetoninc/jdk:1.8
# RUN yum install net-tools -y
COPY be /opt/be
WORKDIR /opt/be
EXPOSE 9050
ENTRYPOINT ["/opt/be/bin/start_be.sh"]

docker build -t doris-be:0.15.0 -f Dockerfile_be .

docker pull bulolo/doris-fe:0.15.0
docker pull bulolo/doris-be:0.15.0

docker run -itd --name fe_1 -p 8030:8030 -p 9030:9030 -v <LOCAL_PATH>/fe_1/conf:/opt/fe/conf -v <LOCAL_PATH>/fe_1/log:/opt/fe/log -v <LOCAL_PATH>/fe_1/doris-meta:/opt/fe/doris-meta doris-fe:0.15.0

docker run -itd --name be_1 -p 9150:9050 -v <LOCAL_PATH>/be_1/conf:/opt/be/conf -v <LOCAL_PATH>/be_1/storage:/opt/be/storage doris-be:0.15.0
docker run -itd --name be_2 -p 9250:9050 -v <LOCAL_PATH>/be_2/conf:/opt/be/conf -v <LOCAL_PATH>/be_2/storage:/opt/be/storage doris-be:0.15.0
docker run -itd --name be_3 -p 9350:9050 -v <LOCAL_PATH>/be_3/conf:/opt/be/conf -v <LOCAL_PATH>/be_3/storage:/opt/be/storage doris-be:0.15.0

version: '3.7'

services:
  doris-fe1:
    image: doris-fe:0.15.0
    container_name: doris-fe1
    ports:
        - 8030:8030
        - 9030:9030
    ulimits:
      nofile:
        soft: "65536"
        hard: "65536"
    volumes:
        - ./fe_1/conf:/opt/fe/conf
        - ./fe_1/log:/opt/fe/log
        - ./fe_1/doris-meta:/opt/fe/doris-meta
    networks:
      doris-network :
        ipv4_address: 172.66.0.100
  doris-be1:
    image: doris-be:0.15.0
    container_name: doris-be1
    ports:
        - 9150:9050
    ulimits:
      nofile:
        soft: "65536"
        hard: "65536"
    volumes:
        - ./be_1/conf:/opt/be/conf
        - ./be_1/log:/opt/be/log
        - ./be_1/storage:/opt/be/storage
    networks:
      doris-network :
        ipv4_address: 172.66.0.101
  doris-be2:
    image: doris-be:0.15.0
    container_name: doris-be2
    ports:
        - 9250:9050
    ulimits:
      nofile:
        soft: "65536"
        hard: "65536"
    volumes:
        - ./be_2/conf:/opt/be/conf
        - ./be_2/log:/opt/be/log
        - ./be_2/storage:/opt/be/storage
    networks:
      doris-network :
        ipv4_address: 172.66.0.102
  doris-be3:
    image: doris-be:0.15.0
    container_name: doris-be3
    ports:
        - 9350:9050
    ulimits:
      nofile:
        soft: "65536"
        hard: "65536"
    volumes:
        - ./be_3/conf:/opt/be/conf
        - ./be_3/log:/opt/be/log
        - ./be_3/storage:/opt/be/storage
    networks:
      doris-network :
        ipv4_address: 172.66.0.103

networks:
  doris-network:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.66.0.0/16
          gateway: 172.66.0.1

vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536

priority_networks = 172.66.0.100/16

priority_networks = 172.66.0.101/16

mysql -P9030 -uroot -p
ALTER SYSTEM ADD BACKEND "172.66.0.101:9050";
ALTER SYSTEM ADD BACKEND "172.66.0.102:9050";
ALTER SYSTEM ADD BACKEND "172.66.0.103:9050";

SET PASSWORD FOR 'root' = PASSWORD('123456'); 

CREATE TABLE mysql_pv_source ( 
  siteid INT, 
  citycode SMALLINT,
  username STRING,
  pv BIGINT
) WITH ( 
  'connector' = 'mysql-cdc', 
  'hostname' = 'localhost', 
  'port' = '3306', 
  'username' = 'root', 
  'password' = 'password', 
  'database-name' = 'demo', 
  'table-name' = 'offices' 
);

CREATE TABLE doris_pv_sink (
    siteid INT, 
    citycode SMALLINT,
    username STRING,
    pv BIGINT
) 
WITH (
  'connector' = 'doris',
  'fenodes' = 'localhost:8030',
  'table.identifier' = 'db_audit.doris_pv',
  'sink.batch.size' = '2',
  'sink.batch.interval'='1',
  'username' = 'root',
  'password' = ''
)

INSERT INTO doris_pv_sink select siteid,citycode,username,pv from mysql_pv_source

docker run -it -v /your/local/.m2:/root/.m2 -v /your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/ apache/incubator-doris:build-env-for-0.15.0

① docker run -it apache/incubator-doris:build-env-for-0.15.0

②wget wget https://dist.apache.org/repos/dist/dev/incubator/doris/0.15/0.15.0-rc04/apache-doris-0.15.0-incubating-src.tar.gz    

这个地址只是0.15版本的doris

或者还可以视同git拉取指定版本的源码

git clone --branch branch-0.15 https://github.com/apache/incubator-doris.git

$ alternatives --set java java-1.8.0-openjdk.x86_64
$ alternatives --set javac java-1.8.0-openjdk.x86_64
$ export JAVA_HOME=/usr/lib/jvm/java-1.8.0

$ alternatives --set java java-11-openjdk.x86_64
$ alternatives --set javac java-11-openjdk.x86_64
$ export JAVA_HOME=/usr/lib/jvm/java-11

sh build.sh

git clone https://github.com/ververica/flink-cdc-connectors.git
cd flink-cdc-connectors
mvn clean install -DskipTests

// 启动集群
bin/start-cluster.sh
// 停止集群
bin/stop-cluster.sh

// 启动集群后，进入flink sql 客户端命令行界面
bin/sql-client.sh embedded

// 表格模式
SET 'sql-client.execution.result-mode' = 'table';
// 变更日志模式
SET 'sql-client.execution.result-mode' = 'changelog';
// Tableau模式
SET 'sql-client.execution.result-mode' = 'tableau';

// 查看当前运行的jobs
bin/flink list
// 查看所有的任务，包括失败、成功、取消的
bin/flink list -a
// 取消命令
bin/flink cancel jobID

SHOW CATALOGS;
SHOW DATABASES;
SHOW TABLES;
SHOW VIEWS;
SHOW FUNCTIONS;
SELECT CURRENT_TIMESTAMP;
RESET table.planner;
RESET
quit

CREATE TABLE datagen (
 f_sequence INT,
 f_random INT,
 f_random_str STRING,
 ts AS localtimestamp,
 WATERMARK FOR ts AS ts
) WITH (
 'connector' = 'datagen',
 -- optional options --
 'rows-per-second'='5',
 'fields.f_sequence.kind'='sequence',
 'fields.f_sequence.start'='1',
 'fields.f_sequence.end'='1000',
 'fields.f_random.min'='1',
 'fields.f_random.max'='1000',
 'fields.f_random_str.length'='10'
);

select * from datagen;

CREATE TABLE employee_information (
    emp_id INT,
    name VARCHAR,
    dept_id INT
) WITH (
    'connector' = 'filesystem',
    'path' = '/path/to/something.csv',
    'format' = 'csv'
);

SELECT * from employee_information WHERE dept_id = 1;

CREATE TABLE offices (
   _id STRING,
   name STRING,
   addr STRING,
   status BOOLEAN,
   PRIMARY KEY (_id) NOT ENFORCED
 ) WITH (
   'connector' = 'mongodb-cdc',
   'hosts' = '10.8.99.44:27011',
   'username' = 'root',
   'password' = '@junyao2022',
   'database' = 'biocitydb',
   'collection' = 'offices'
 );

select * from offices;

CREATE TABLE products (
  id INT,
  name STRING,
  description STRING
) WITH (
  'connector' = 'mysql-cdc',
  'hostname' = 'localhost',
  'port' = '3306',
  'username' = 'root',
  'password' = '123456',
  'database-name' = 'mydb',
  'table-name' = 'products'
);

CREATE TABLE shipments (
  shipment_id INT,
  order_id INT,
  origin STRING,
  destination STRING,
  is_arrived BOOLEAN
) WITH (
  'connector' = 'postgres-cdc',
  'hostname' = 'localhost',
  'port' = '5432',
  'username' = 'postgres',
  'password' = 'postgres',
  'database-name' = 'postgres',
  'schema-name' = 'public',
  'table-name' = 'shipments'
);

CREATE TABLE material (
    FMATERIALID INT,
    FNUMBER STRING,
    PRIMARY KEY (FMATERIALID) NOT ENFORCED
  ) WITH (
    'connector' = 'sqlserver-cdc',
    'hostname' = '10.8.99.31',
    'port' = '1433',
    'username' = 'sa',
    'password' = 'XXXsa1999$',
    'database-name' = 'dkw',
    'schema-name' = 'dbo',
    'table-name' = 'T_BD_MATERIAL'
  );

select * from material;

CREATE TABLE enriched_orders (
  order_id INT,
  order_date TIMESTAMP_LTZ(3),
  customer_id INT,
  price DECIMAL(10, 5),
  product ROW<name STRING, description STRING>,
  order_status BOOLEAN,
  customer_name STRING,
  customer_address STRING,
  PRIMARY KEY (order_id) NOT ENFORCED
) WITH (
 'connector' = 'elasticsearch-7',
 'hosts' = 'http://localhost:9200',
 'index' = 'enriched_orders',
 'username' = 'root',
 'password' = 'password'
);

CREATE TABLE doris_test_sink (
   id INT,
   name STRING
) 
WITH (
  'connector' = 'doris',
  'fenodes' = 'localhost:8030',
  'table.identifier' = 'db_audit.doris_test',
  'sink.batch.size' = '2',
  'sink.batch.interval'='1',
  'username' = 'root',
  'password' = ''
)

INSERT INTO department_counts
SELECT 
   dept_id,
   COUNT(*) as emp_count 
FROM employee_information;

{
    "_id": ObjectId("61ea6ee776d12fd2c261c105"),
    "fileName": "2.xlsx",
    "creator": ObjectId("5eba176654c70a2bc8df0719"),
    "uploadDate": ISODate("2022-01-21T08:29:27.823Z"),
    "creatorName": "张云",
}

{
    "_id": ObjectId("61ea6d78703190d0efca84df"),
    "org": "DKWSH",
    "contact": "翁文斌",
    "salesId": ObjectId("5ef3214dc15bc674d6976365"),
    "remark": null,
    "items": [
        {
            "detailId": "MX21305683",
            "materialName": "PerCP anti-human CD11c",
            "spec": "100 tests",
            "itemNum": "337234",
            "batchNumber": "B332675",
            "amount": NumberInt("6"),
            "manufacturer": "Biolegend"
        }
    ],
    "file": ObjectId("61ea6ee776d12fd2c261c105"),
}

// 创建文件
const file = await ctx.model.Delivery.File.create(files)

// 创建订单
const orders = await ctx.model.Delivery.VirtualOrder.create(orders)

// 发送通知
const sendRes = await ctx.app.noticeQueue.addBulk(datas)

// 1.校验是否开过发票和发票池

// 2.创建开票池
const invoiceItemOpts = {
    org: 'XXX',
    order: 'XXX',
    detailId: 'XXX',
    materialName: 'XXX',
  }
const invoiceItems = await ctx.model.Delivery.InvoiceItem.create(
  invoiceItemOpts
)

// 3.创建发票
let invoiceOpts = {
  invoiceNum: ctx.helper.createInvoiceNum('P', parseInt(next_invoice_num)),
}
const invoice = await ctx.model.Delivery.Invoice.create(invoiceOpts)

2022-02-21 10:58:17,497 ERROR 30265 [-/116.233.76.38/-/25ms POST /orders/6212ff47fa481876394ee21c/status] error_handler: MongoError: E11000 duplicate key error collection: biocitydb.sys_invoices index: invoiceNum_1 dup key: { invoiceNum: "P2202211321" }

2022-02-21 10:57:38,020 INFO 30265 发票invoiceOpts {
  invoiceNum: 'P2202211296'
  saleName: '王璐',
}

2022-02-21 10:57:38,022 INFO 30265 发票invoiceOpts {
  invoiceNum: 'P2202211296',
  saleName: '沈梦婷',
}

# 启用调试日志
verbose = true

mongo-url = "mongodb://root:<password>@10.8.99.44:27011/?authSource=admin" 
elasticsearch-urls = ["https://<host>:9200"]

# index GridFS files inserted into the following collections
file-namespaces = ["biocitydb.materials"]
# 此选项允许你直接将集合从 MongoDB 复制到 Elasticsearch。 Monstache 允许过滤实际索引到 Elasticsearch 的数据，因此你不一定需要复制整个集合。 在上面，我们同步数据库 test 中的 mycol 集合。
direct-read-namespaces = ["biocitydb.materials"]
# 实时通知以告知 Elasticsearch 所有写入文档，包括指定集合中的删除和更新。
change-stream-namespaces = ["biocitydb.materials"] 

namespace-regex = '^biocitydb\.materials$'

# 压缩请求到es
gzip = true

# generate indexing statistics
stats = true

# index statistics into Elasticsearch
index-stats = true

elasticsearch-user = "elastic"
elasticsearch-password = "<password>"

#monstache最多开几个线程同步到es,默认为4
elasticsearch-max-conns = 2 

# 证书文件
elasticsearch-pem-file = "/monstache/client.crt.pem"
elasticsearch-validate-pem-file = false

# mongodb删除集合或库时是否同步删除es中的索引
dropped-collections = true
dropped-databases = false

# 更新es而不是覆盖
index-as-update = true

replay = false

# 记录同步位点，便于下次从该位置同步
resume = true

# do not validate that progress timestamps have been saved
resume-write-unsafe = false

# 需要es ingest-attachment 
index-files = false

# turn on search result highlighting of GridFS content
file-highlighting = true

# 高可用模式下需要配置集群名称，集群名称一样的进程会自动加入一个集群内,这个是monstance的集群，不是es
cluster-name = '<name>'

# do not exit after full-sync, rather continue tailing the oplog
exit-after-direct-reads = false

# 生产环境以日志文件输出，默认以命令行输出
# [logs]
# info = "./logs/info.log"
# warn = "./logs/wran.log"
# error = "./logs/error.log"
# trace = "./logs/trace.log"

# mapping定义mongodb数据到es的索引名称和type，namespace是库名.集合名
# 这里需要注意一件事：最好是在es中创建好你要的索引结构，关闭es的自动创建索引功能
[[mapping]]
namespace = "biocitydb.materials" 
index = "materials"
 
[[script]]
namespace = "biocitydb.materials"
path = "./scripts/materials.js"
routing = true

module.exports = function (doc) {
    delete doc._id;
    //TODO
    return doc;
}

-new    :说明生成证书请求文件
-x509   :说明生成自签名证书
-key    :指定已有的秘钥文件生成秘钥请求，只与生成证书请求选项-new配合。
-newkey :-newkey是与-key互斥的，-newkey是指在生成证书请求或者自签名证书的时候自动生成密钥，
         然后生成的密钥名称由-keyout参数指定。当指定newkey选项时，后面指定rsa:bits说明产生
         rsa密钥，位数由bits指定。 如果没有指定选项-key和-newkey，默认自动生成秘钥。
-out    :-out 指定生成的证书请求或者自签名证书名称
-days   :证书的有效期限；
-config :默认参数在ubuntu上为 /etc/ssl/openssl.cnf, 可以使用-config指定特殊路径的配置文件
-nodes  :如果指定-newkey自动生成秘钥，那么-nodes选项说明生成的秘钥不需要加密，即不需要输入passphase.  
-batch  :指定非交互模式，直接读取config文件配置参数，或者使用默认参数值  

// 生成私钥(key文件)
openssl genrsa -out ca.key 2048
// 使用私钥ca.key生成签名请求(csr文件) CSR 即证书签名申请（Certificate Signing Request）
openssl req -new -key ca.key -out ca.csr
// 使用签名请求生成根证书(crt文件)
openssl x509 -req -days 365 -in ca.csr -signkey ca.key -out ca.crt

// 生成私钥ca.key
openssl genrsa -out ca.key 2048
// 使用私钥ca.key请求并生成根证书pem格式
openssl req -x509 -new -nodes -key ca.key -sha256 -days 365 -out ca.pem

// 生成私钥
openssl genrsa -out server.key 2048
// 生成证书请求文件
openssl req -new -key server.key -out server.csr
// 使用 CA 的根证书为服务器证书签名
openssl x509 -req -in server.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256

-----BEGIN CERTIFICATE-----

Base64编码过的证书数据

-----END CERTIFICATE-----

// .pem -> .pfx
openssl pkcs12 -export -in cert.pem -out cert.pfx -inkey key.pem 
// .pfx -> .cer
openssl pkcs12 -in server.pfx -out server.cer -nodes
// .cer -> .pem
openssl x509 -inform der -in server.cer -out server.pem
// PEM -> DER
openssl x509 -in server.pem -outform der -out server.der
// DER -> PEM
openssl x509 -in server.der -inform der -outform pem -out server.pem