俊瑶先森

docker exec -it docker_logstash bash

./bin/logstash-plugin install logstash-input-jdbc

input{
     jdbc{
          # 连接数据库
          jdbc_connection_string => "jdbc:mysql://47.119.168.111:3306/fob?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false"
          jdbc_user => "root"
          jdbc_password => "XXXXX"
          # 连接数据库的驱动包
          jdbc_driver_library => "/usr/share/logstash/config/mysql/mysql-connector-java-8.0.24.jar"
          jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
          jdbc_paging_enabled => "true"
          jdbc_page_size => "50000"
          codec => plain { charset => "UTF-8" }

          # 数据追踪
          # 追踪的字段
          tracking_column => "updated_at"
          # 上次追踪的元数据存放位置
          last_run_metadata_path => "/usr/share/logstash/config/lastrun/logstash_jdbc_last_run"
          # 设置时区
          jdbc_default_timezone => "Asia/Shanghai"  
          # sql 文件地址
          # statement_filepath => ""
          # sql
          statement => "SELECT g.id AS id,g.product_name AS product_name,g.shop_id AS shop_id,g.category_id AS category_id,g.keyword AS keyword,g.status AS status FROM fg_product g WHERE g.updated_at > :sql_last_value"
          # 是否清除 last_run_metadata_path 的记录,如果为真那么每次都相当于从头开始查询所有的数据库记录
          clean_run =>false
          # 这是控制定时的，重复执行导入任务的时间间隔，第一位是分钟 不设置就是1分钟执行一次
          schedule => "* * * * *"
     }
}
output{
      elasticsearch{
                   # 要导入到的Elasticsearch所在的主机
                   hosts => "47.119.168.111:9200"
                   # 要导入到的Elasticsearch的索引的名称
                   index => "fob_index"
                   # 类型名称（类似数据库表名）
                   document_type => "fg_product"
                   # 主键名称（类似数据库表名）
                   document_id => "%{id}"
      }
      
      stdout{
            # JSON 格式输出
            codec => json_lines
      } 
}


查看mysql数据是否进入到es

{
    "name": "5f04c6c0a818",
    "cluster_name": "elasticsearch",
    "cluster_uuid": "vUPjZeMvSGqO_lrtmhqlmw",
    "version": {
        "number": "7.12.0",
        "build_flavor": "default",
        "build_type": "docker",
        "build_hash": "78722783c38caa25a70982b5b042074cde5d3b3a",
        "build_date": "2021-03-18T06:17:15.410153305Z",
        "build_snapshot": false,
        "lucene_version": "8.8.0",
        "minimum_wire_compatibility_version": "6.8.0",
        "minimum_index_compatibility_version": "6.0.0-beta1"
    },
    "tagline": "You Know, for Search"
}

health status index                           uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   bank                            ktIRbx9ZTwyDrEHlxiZpqw   1   1       1000            0    379.3kb        379.3kb
green  open   .kibana_task_manager_7.12.0_001 Yi3dSuiVSYWYaFidZxHSJg   1   0          9        26006      2.6mb          2.6mb
green  open   .apm-custom-link                DLOQIooBRiWD4O237c8tBA   1   0          0            0       208b           208b
green  open   .apm-agent-configuration        zUC91G_oRw63OY54rV2orw   1   0          0            0       208b           208b
green  open   .async-search                   NkCazI4YQDirjCrnvlEv2Q   1   0          0           24    794.5kb        794.5kb
green  open   .kibana_7.12.0_001              z4vax_yNTq2cOP7JxBphOQ   1   0         63           10      2.1mb          2.1mb
green  open   .kibana-event-log-7.12.0-000001 CYz7KhrsRa-dino1NEPDog   1   0          7            0     32.9kb         32.9kb
green  open   .tasks                          6wSie0zSRIW17VNQuxHgNQ   1   0          8            0     42.4kb         42.4kb

{"index":{"_id":"1"}}
{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"}
{"index":{"_id":"6"}}
{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"}
{"index":{"_id":"13"}}
{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA"}

{
    "took": 1,
    "timed_out": false,
    "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 61,
            "relation": "eq"
        },
        "max_score": 1.0,
        "hits": [
            {
                "_index": "bank",
                "_type": "account",
                "_id": "51",
                "_score": 1.0,
                "_source": {
                    "account_number": 51,
                    "balance": 14097,
                    "firstname": "Burton",
                    "lastname": "Meyers",
                    "age": 31,
                    "gender": "F",
                    "address": "334 River Street",
                    "employer": "Bezal",
                    "email": "burtonmeyers@bezal.com",
                    "city": "Jacksonburg",
                    "state": "MO"
                }
            }
        ]
    }
}

{
    "query": {
        "bool": {
            "must": {
                "match_all": {}
            },
            "filter": {
                "range": {
                    "balance": {
                        "gte": 20000,
                        "lte": 30000
                    }
                }
            }
        }
    },
    "sort": {
        "age": {
            "order": "asc"
        }
    },
    "_source": [
        "account_number",
        "balance",
        "address"
    ],
    "from": 1,
    "size": 10
}

{
    "took": 1,
    "timed_out": false,
    "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 217,
            "relation": "eq"
        },
        "max_score": null,
        "hits": [
            {
                "_index": "bank",
                "_type": "account",
                "_id": "292",
                "_score": null,
                "_source": {
                    "account_number": 292,
                    "address": "691 Nassau Street",
                    "balance": 26679
                },
                "sort": [
                    20
                ]
            }
        ]
    }
}

GET vpn-log-*/_search
{
  "query": {
    "bool": {
      "must": {
        "match": {
          "_index": "<vpn-log-{now/d{YYYY-MM-dd}}>"
        }
      },
      "filter": [
        {
          "terms": {
            "tags": [
              "QN"
            ]
          }
        }
      ]
    }
  },
  "aggs": {
    "topn": {
      "terms": {
        "field": "tags"
      },
      "aggs": {
        "source_ip_topn": {
          "terms": {
            "field": "source_ip"
          }
        }
      }
    }
  }
}

docker run --name elasticsearch -p 9200:9200 -p 9300:9300 \
-e ES_JAVA_OPS="-Xms256m -Xmx256m" \
-e "discovery.type=single-node" \
--network elastic_stack \
-d docker.elastic.co/elasticsearch/elasticsearch:7.12.0

// 把配置文件拿出来
docker cp elasticsearch:/usr/share/elasticsearch/config /data/dockers/es/config
docker cp elasticsearch:/usr/share/elasticsearch/data /data/dockers/es/data
docker cp elasticsearch:/usr/share/elasticsearch/plugins /data/dockers/es/plugins

docker run --name elasticsearch -p 9200:9200 -p 9300:9300 \
-e ES_JAVA_OPS="-Xms256m -Xmx256m" \
-e "discovery.type=single-node" \
--network elastic_stack \
-v /data/dockers/es/config:/usr/share/elasticsearch/config \
-v /data/dockers/es/data:/usr/share/elasticsearch/data \
-v /data/dockers/es/plugins:/usr/share/elasticsearch/plugins \
-d docker.elastic.co/elasticsearch/elasticsearch:7.12.0

docker run --name kibana \
--network elastic_stack \
-p 5601:5601 -d docker.elastic.co/kibana/kibana:7.12.0

docker run --name logstash \
-d docker.elastic.co/logstash/logstash:7.12.0

docker logs -f logstash

docker cp logstash:/usr/share/logstash/config /data/dockers/logstash/config

docker cp logstash:/usr/share/logstash/data /data/dockers/logstash/data

docker cp logstash:/usr/share/logstash/pipeline /data/dockers/logstash/pipeline

mkdir -p /data/dockers/logstash/config/conf.d

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
path.config: /usr/share/logstash/config/conf.d/*.conf  //配置mysql同步es配置使用

chmod -R 777 /data/dockers/logstash/

docker run \
--name logstash \
--restart=always \
--network elastic_stack \
-p 5044:5044 \
-p 9600:9600 \
-e ES_JAVA_OPTS="-Duser.timezone=Asia/Shanghai" \
-v /data/dockers/logstash/config:/usr/share/logstash/config \
-v /data/dockers/logstash/data:/usr/share/logstash/data \
-v /data/dockers/logstash/pipeline:/usr/share/logstash/pipeline \
-d logstash:7.12.0

yarn global add @nestjs/cli
nest new nestjs-admin

./nestjs-admin
├── .gitignore
├── .prettierrc
├── README.md
├── nest-cli.json
├── package.json
├── src
│   ├── app.controller.spec.ts
│   ├── app.controller.ts
│   ├── app.module.ts
│   ├── app.service.ts
│   └── main.ts
├── test
│   ├── app.e2e-spec.ts
│   └── jest-e2e.json
├── tsconfig.build.json
├── tsconfig.json
├── tslint.json
└── yarn.lock

node_modules  
.git  
.idea  
.vscode  
/coverage  

FROM node:12-alpine AS dependencies  
WORKDIR /usr/src/app  
COPY package.json yarn.lock ./  
RUN yarn install --production  
  
FROM node:12-alpine  
WORKDIR /usr/src/app  
COPY package.json dist ./  
COPY --from=dependencies /usr/src/app/node_modules ./node_modules  
EXPOSE 3000  
CMD [ "node", "dist/main" ]  

"build:docker": "yarn build && docker build -t bulolo/$npm_package_name:latest . && docker push bulolo/$npm_package_name:latest",

pids like {pids},{id}%

nest g mo user
nest g controller user
nest g service user --no-spec

"db": "rm -rf src/entities & npx typeorm-model-generator -h localhost -d race -p 3306 -u root -x hong -e mysql -o src/entities --noConfig --relationIds --lazy --ce none --cp none"

-h dbhost
-d dbname
-p port
-u username
-x password
-e dbtype

-a 生成实体类 extends BaseEntity 可使用基础的CURD
-o 默认输出项目根目录下 output 文件夹下 可以自定义输出目标目录，例：-o ./src/demo/entities
--cf，将文件名转换为指定的大小写 [可选值："pascal"，"camel"，"none"] [默认值 none"] 
--ce pascal 生成实体类名采用人名式格式 [可选值："pascal"，"camel"，"none"] [默认值 none"] 
--cp camel 生成实体类属性采用驼峰 [可选值："pascal"，"camel"，"none"] [默认值 none”] 
--noConfig 不产生相关配置文件

npm run db

npm run db -- --tables sys_user,sys_role

import { Injectable, NestMiddleware } from '@nestjs/common';
import { Request, Response } from 'express';

@Injectable()
export class LoggerMiddleware implements NestMiddleware {
  use(req: Request, res: Response, next: Function) {
    const logFormat = `  >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
      Request original url: ${req.originalUrl}
      Method: ${req.method}
      IP: ${req.ip}
      Status code: ${res.statusCode}
      Parmas: ${JSON.stringify(req.params)}
      Query: ${JSON.stringify(req.query)}
      Body: ${JSON.stringify(req.body)} \n  >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> 
    `
    console.log(logFormat);
    next();
  }
}

import { LoggerMiddleware } from './common/middleware/logger.middleware'
app.use(new LoggerMiddleware().use)

for (let index of ['a', 'b'].keys()) {
  console.log(index);
}
// 0
// 1

for (let item of ['a', 'b'].values()) {
  console.log(item);
}
// 'a'
// 'b'

for (let [index, value] of ['a', 'b'].entries()) {
  console.log(index, value);
}
// 0 "a"

// 对象转数组
const obj = {
    a: '1',
    b: 2
}
const arr = Object.entries(obj)
console.log(arr) // [ [ 'a', '1' ], [ 'b', 2 ] ]

// 数组转对象
const obj = Object.fromEntries([
    ['a', '1'],
    ['b', 2]
])
console.log(obj) //{ a: '1', b: 2 }

const array = [1,2].concat(['a', 'b'], ['name']);
// [1, 2, "a", "b", "name"]

const arr1 = [1, 2]
const arr2 = ['a', 'b']
const array = [...arr2, arr2]

array.indexOf(item,start)

var arr = ['apple','orange','pear'];
console.log("found:", arr.indexOf("orange") != -1);

array.forEach(function(currentValue, index, arr), thisValue)

let arr1 = [1, 5, 7, 8, 9, 6]
let str = ''

arr1.forEach((item, index, array) => {
  console.log(item, index, array)
  str += `<li>${item}</li>`
})
console.log(str)

array.some(function(currentValue,index,arr),thisValue)
array.every(function(currentValue,index,arr), thisValue)

let arr = [5, 7, 8, 9, 6, ]
const v = arr.some(item => {
    return item < 6
})
const x = arr.every(item => {
    return item < 9
})

console.log(v) //true
console.log(x) //false

const obj = [{
        name: '小明',
        age: 9,
        sex: '男'
    },
    {
        name: '小红',
        age: 10,
        sex: '女'
    }
]
const a = obj.some(item => {
    return item.name == '小红'
})
const b = obj.every(item => {
    return item.age == 9
})
console.log(a) //true
console.log(b) //false

array.filter(function(currentValue,index,arr), thisValue)

let arr = [1, 5, 6, 3, 4, 2, 8, 7];
const a = arr.filter(item => {
    return item < 5
});
console.log(a) //[1,3,4,2]

var array = [1, 2, 1, 2, 4, 5, 3, 5, 6, 7, 6];
// 检测数组下标，是否等于当前项的下标，不相等则认为有重复
var result = array.filter((item, index, arr) => {
    // item : 数组每一项的值
    // index: 每一项的下标
    // arr: 原始数组
    return index === arr.indexOf(item);
})

console.log(result); //[1, 2, 4, 5,3, 6, 7]

array.find(function(currentValue, index, arr),thisValue)

let arr = [1, 5, 6, 3, 4, 2, 8, 7];
  const b = arr.find(item => {
      return item > 9
  });
  const c = arr.find(item => {
      return item < 9
  });
  console.log(b) //undefined
  console.log(c) //1

arr.includes(searchElement)
arr.includes(searchElement, fromIndex)

let arr = [1, 5, 6, 3, 4, 2, 8, 7];
const d = arr.includes(7)
const x = arr.includes(9)
console.log(d) //true
console.log(x) //false

array.map(function(currentValue,index,arr), thisValue)

let arr = [1, 5, 6, 3, 4, 2, 8, 7];
 const y = arr.map((item, index, array) => {
     return item, array
 })
 console.log(y)
 //   (8)[Array(8), Array(8), Array(8), Array(8), Array(8), Array(8), Array(8), Array(8)]
 //   0: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   1: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   2: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   3: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   4: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   5: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   6: (8)[1, 5, 6, 3, 4, 2, 8, 7]
 //   7: (8)[1, 5, 6, 3, 4, 2, 8, 7]

array.reduce(function(total, currentValue, currentIndex, arr), initialValue)

const arr = [3,9,4,3,6,0,9]
const sum = arr.reduce((pre, item) => {
    return pre + item
}, 0)
console.log(sum) // 34

const arr = [3,9,4,3,6,0,9]
var max = arr.reduce((prev, cur) => {
    return Math.max(prev,cur)
})
console.log(max) // 9

const arr = [3,9,4,3,6,0,9]
var newArr = arr.reduce((prev, cur) => {
    prev.indexOf(cur) === -1 && prev.push(cur);
    return prev;
},[]);
console.log(newArr) // [ 3, 9, 4, 6, 0 ]

let person = [
     {id: 0, name: "小明"},
     {id: 1, name: "小张"},
     {id: 2, name: "小李"},
     {id: 3, name: "小孙"},
     {id: 1, name: "小周"},
     {id: 2, name: "小陈"},  
];
 
let obj = {};
 
let peon = person.reduce((cur,next) => {
    obj[next.id] ? "" : obj[next.id] = true && cur.push(next);
    return cur;
},[]) //设置cur默认类型为数组，并且初始值为空的数组
console.log(peon);

array.sort(sortfunction)

var points = [40, 100, 1, 5, 25, 10]
points.sort( (a, b) => {
    return a - b
})

const arr = [3,9,4,3,6,0,9]
let set = new Set(arr)
const newArr = Array.from(set)
console.log(newArr) // [ 3, 9, 4, 6, 0 ]