yq-v4

moon / 105 / 2023-08-29 12:03:12

ChatGPT 可用网址，仅供交流学习使用，如对您有所帮助，请收藏并推荐给需要的朋友。
https://ckai.xyz

一. 安装

安装 yq 4 版本

yq 解析 YAML 文件的方式类似于树解析，将 YAML 文件划分为不同的节点类型，这些节点类型相互嵌套

wget https://github.com/mikefarah/yq/releases/download/v4.33.1/yq_linux_amd64

chmod +x yq_linux_amd64 && cp yq_linux_amd64 /usr/bin/yq

[root@yunwei-k8s-addon1-test xingguang]# yq -V
yq (https://github.com/mikefarah/yq/) version v4.33.2

二. yaml 案例

1. coco.yaml

kind: Service
apiVersion: v1
metadata:
  name: prometheus
  namespace: kube-system
  labels:
    app.kubernetes.io/name: prometheus
spec:
  type: ClusterIP
  clusterIP: None
  selector:
   app.kubernetes.io/name: prometheus
  ports:
    - name: web
      protocol: TCP
      port: 8080
      targetPort: web
    - name: grpc
      port: 10901
      targetPort: grpc
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app.kubernetes.io/name: prometheus
  name: prometheus-rep-0
  namespace: kube-system
spec:
  podManagementPolicy: Parallel
  replicas: 0
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
      kvass/rep: "0"
  serviceName: prometheus
  template:
    metadata:
      labels:
        app.kubernetes.io/name: prometheus
        kvass/rep: "0"
    spec:
      containers:
        - name: thanos
          image: thanosio/thanos:v0.18.0
          args:
            - sidecar
            - --tsdb.path=/prometheus
            - --prometheus.url=http://localhost:8080
            - --reloader.config-file=/etc/prometheus/config/prometheus.yml
            - --reloader.config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml
          ports:
            - name: http-sidecar
              containerPort: 10902
            - name: grpc
              containerPort: 10901
          livenessProbe:
            httpGet:
              port: 10902
              path: /-/healthy
          readinessProbe:
            httpGet:
              port: 10902
              path: /-/ready
          volumeMounts:
            - mountPath: /etc/prometheus/config_out
              name: config-out
            - mountPath: /etc/prometheus/config
              name: config
            - mountPath: /etc/localtime
              name: time
        - name: kvass
          args:
            - sidecar
            - --store.path=/prometheus/ # where to store kvass local data
            - --config.file=/etc/prometheus/config_out/prometheus.env.yaml # origin config file
            - --config.output-file=/etc/prometheus/config_out/prometheus_injected.yaml # injected config file. this is the file prometheus use
          image: tkestack/kvass:v0.3.1
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - mountPath: /etc/prometheus/config_out
              name: config-out
            # sidecar need pvc to store targets list， see '--store.path" flag
            # sidecar will reload targets list in initialization phase
            - mountPath: /prometheus
              name: data
            - mountPath: /etc/localtime
              name: time
          ports:
            - containerPort: 8080
              name: web
              protocol: TCP
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
        - name: prometheus
          args:
            - --storage.tsdb.path=/prometheus
            - --storage.tsdb.retention.time=15d
            - --web.enable-lifecycle
            - --storage.tsdb.no-lockfile
            - --storage.tsdb.max-block-duration=2h
            - --storage.tsdb.min-block-duration=2h
            - --config.file=/etc/prometheus/config_out/prometheus_injected.yaml # use injected config file instead of origin config file
            - --log.level=debug
          image: prom/prometheus:v2.33.3
          ports:
            - containerPort: 9090
              name: server
              protocol: TCP
          volumeMounts:
            - mountPath: /etc/localtime
              name: time  
            - mountPath: /etc/prometheus/config
              name: config
            - mountPath: /etc/prometheus/config_out
              name: config-out
            - mountPath: /prometheus
              name: data
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      serviceAccountName: prometheus
      securityContext:
        runAsUser: 0
      volumes:
#        - name: data
#          emptyDir: {}
        - name: time
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: config
          configMap:
            name: prometheus-config
            defaultMode: 420
        - emptyDir: {}
          name: config-out
        - emptyDir: {}
          name: tls-assets
  volumeClaimTemplates:
    - metadata:
        labels:
          k8s-app: prometheus
        name: data
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 3Gi
        storageClassName: managed-nfs-storage
        volumeMode: Filesystem
  updateStrategy:
    rollingUpdate:
      partition: 0
    type: RollingUpdate

2. 1-backup.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: zjs
  name: zjs-epidemic-dubbo
spec:
  progressDeadlineSeconds: 100
  replicas: 1
  revisionHistoryLimit: 1
  selector:
    matchLabels:
      service/env: "test"
      name: "zjs-epidemic-dubbo"
  template:
    metadata:
      labels:
        service/env: "test"
        name: "zjs-epidemic-dubbo"
        k8s.kuboard.cn/layer: svc
    spec:
      imagePullSecrets:
      - name: harborsecret
      containers:
      - image: 'harbor.yeemiao.net.cn/zjs/zjs-epidemic-dubbo:20220507160701'
        imagePullPolicy: IfNotPresent
        name: zjs-epidemic-dubbo
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        command:
        - java
        - '-XX:MinRAMPercentage=30.0'
        - '-Xss256k'
        - '-XX:MaxRAMPercentage=100.0'
        - '-Dproject.name=zjs-epidemic-dubbo'
        - '-Dpinpoint.applicationName=zjs-epidemic-dubbo'
        - '-Dpinpoint.agentId=$(agentID)_17963'
        - >-
          -javaagent:/home/yeemiao/pinpoint/pinpoint-agent-2.2.0/pinpoint-bootstrap-2.2.0.jar
        - '-jar'
        - threegene-boot.jar
        env:
        - name: agentID
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
        - name: DUBBO_IP_TO_REGISTRY
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: DUBBO_PORT_TO_REGISTRY
          value: '17963'
        - name: DUBBO_PORT_TO_BIND
          value: '17963'
        ports:
        - containerPort: 17963
          hostPort: 17963
        resources:
          limits:
            cpu: 4000m
            memory: 3000Mi
          requests:
            cpu: 100m
            memory: 1024Mi
        livenessProbe: # 如果探测失败会重启容器
          tcpSocket: # 通过tcp协议对端口进行检测如果端口可以连通就视为检测成功
            port: 17963
          # 检测参数配置
          initialDelaySeconds: 30 # 初始延迟秒数，也就容器启动多久后开始检测
          timeoutSeconds: 30 # 响应超时时间
          periodSeconds: 30 # 检测周期，也就检测时间间隔
        readinessProbe:
          exec:
            command:
            - /bin/bash
            - /home/health.sh
          initialDelaySeconds: 35
          timeoutSeconds: 2 # 响应超时时间
          periodSeconds: 10
        volumeMounts:
        - name: apollo-config
          mountPath: /opt/settings
        - name: jdk
          mountPath: /usr/local/java
        - name: pinpoint
          mountPath: /home/yeemiao/pinpoint
        - name: logs
          mountPath: /home/yeemiao/logs
      volumes:      
      - name: apollo-config
        configMap:
          name: test-apollo-config
      - name: jdk
        hostPath:
          path: /usr/local/java
      - name: pinpoint
        hostPath:
          path: /home/yeemiao/pinpoint
      - name: logs
        hostPath:
          path: /var/log/containers/baselogs
          type: DirectoryOrCreate 
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      terminationGracePeriodSeconds: 10

3. pod.yaml

apiVersion: v1
kind: Pod
metadata:
  name: test-pod
  labels:
    l1: v1
    app.name: test-pod
  annotations:
    com.team.owner: abc
spec:
  volumes:
  - name: html
    emptyDir: {}
  containers:
  - name: c1
    image: nginx
    volumeMounts:
    - name: html
      mountPath: /usr/share/nginx/html
  - name: c2
    image: debian
    volumeMounts:
    - name: html
      mountPath: /html
    command: ["/bin/sh", "-c"]
    args:
      - while true; do
          date >> /html/index.html;
          sleep 1;
        done

三. 使用案例

1. 标量类型查询

使用的是 RHS 表达式

# 修改 namespace
yq  -i '.metadata.namespace = "xingguang"' 1-backup.yaml

# 匹配 image，从4.18版本开始默认可以不用加 -r 参数， yq默认为 eval 命令， [0]： 表示数组中的第一个元素，下面匹配的就是第一个容器
yq -r '.spec.template.spec.containers.[0].image' 1-backup.yaml
yq '.spec.template.spec.containers.[2].name' coco.yaml
yq eval '.spec.template.spec.containers.[1].args' coco.yaml

2. Map-Array类型

数组和列表

### 数组相关, 匹配数组中第一个元素
yq .spec.template.spec.containers[0].command.[0] 1-backup.yaml

# 匹配数组中所有内容
yq .spec.template.spec.containers[0].command.[] 1-backup.yaml

# 数组中有多个元素 可以使用 select 选择，也可以用通配符匹配
yq '.spec.template.spec.containers[]  | select(.name == "prometheus") | .image' coco.yaml

# 如果yaml文件有多个kind，修改
yq 'select(.kind == "Deployment") | .spec.template.spec.containers[0].resources.limits.cpu = "1"' allinone.yaml

# 用通配符匹配匹配, select 可以用 >、<、== 比较运算
yq '.spec.template.spec.containers[]  | select(.name == "*us") | .image' coco.yaml

# 复杂场景可以使用正则方程去匹配： test, match,capture
yq '.spec.template.spec.containers[]  | select(.image | test("thanos")) | .image' coco.yaml
yq '.spec.template.spec.containers[]  | select(.image | match("thanos")) | .image' coco.yaml
yq '.spec.template.spec.containers[]  | select(.image | capture("thanos")) | .image' coco.yaml

3. 变量引用

variable，定义了要在后续管道中使用的变量
- strenv
- env

# 使用 env(变量名) 调用， 可动态使用
ns='hhy' ys='xingguang' yq  -i '.metadata.namespace = env(ns)' 1-backup.yaml

# 如何比较不同层之间的内容， 验证是否与容器中的volumeMounts一致， spec.volumes， 先通过 as 进行赋值给变量 $volumeName
yq '.spec | .volumes[].name as $volumeName | .containers[] | select(.volumeMounts[] | .name == $volumeName)' pod.yaml
yq '.spec | .volumeClaimTemplates[].metadata.name as $ya | .template.spec.containers[] | select(.volumeMounts[] | .name == $ya)' coco.yaml

parent ，可以返回前一层中的节点内容

yq '.spec.template.spec.containers[] | .args |  parent' coco.yaml

4. 增，删，改

-i/–inplace. 默认情况下，结果被发送到标准输出，对当前文件的修改可以用-i. 而我们需要文件重定向 operator( >) 来输出到一个新文件，如果更新的字段不存在，则有 update 操作变为 add
-o/–output-format. 此输出格式默认为 YAML，但也支持json/j, xml/x, 和props/p
-I/–indent, YAML 的缩进。默认为2，这也是YAML规范的标准
–from-file, 从文件中读取表达式

运算符：|=

RHS 表达式是在：每个 LHS 节点作为上下文运行时运行的，对于基于旧值更新值很有用，例如增量

# 不加 -i 就预览yaml，不会真正修改，加上 -i 才会修改
# 用 |= 表达式不是替换某个value，而是增加内容
yq '.metadata.name |= . + "abc"' coco.yaml

# |= 配合变量
USER="chenxingguang"; yq '.metadata.name |= strenv(USER) + "-" + .' pod.yaml

1). 增

# 直接在spec下面增加一个键值对
yq '.spec = {"nodeName":"master"} + .spec ' coco.yaml

yq '.spec.template.spec.containers[] | select(.name == "zjs*") | .env[] | select(.name == "agentID") | .valueFrom.fieldRef |= {"addkey":"addvalue"} + .' 1-backup.yaml

# 给 yaml 配置注释
yq '.metadata.annotations["com.team.owner"] line_comment="yaya-0304"' pod.yaml

2）改

# 同时修改两个容器的镜像拉取策略为
yq 'with(.spec.containers[]; .imagePullPolicy="Never")' pod.yaml
yq 'with(.spec.template.spec.containers[]; .imagePullPolicy="Never")' coco.yaml

3). 删除

删除虽然可以单独使用，但一般与查询、条件查询结合使用，支持删除的运算符包括 substract 和 del

# 删除 nginx 容器
yq '.spec.containers - [{"image":"nginx", "name":"c1", "volumeMounts":[{"name":"html","mountPath":"/usr/share/nginx/html"}]}]' pod.yam

# 简化版删除容器
yq 'del(.spec.containers[] | select(.image == "debian"))' pod.yaml


# 删除一个键值对
yq -i 'del(.spec.nodeName)' pod.yaml

# 输出删除列表中的其中一条
yq '.spec.template.spec.containers[].command - ["-Dpinpoint.agentId=$(agentID)_17963"]' 1-backup.yaml

# 复杂场景的的修改一般会结合起来使用
yq -e -i '.spec.template.spec.containers[] | .command - ["-Dpinpoint.agentId=$(agentID)_17963"] as $l | .command = $f' 1-backup.yaml

# 批量删除， yaml以 team{1,2,3} 开头的, 删除
yq -i '["team1*", "team2*", "team3*"] as $names | select(.kind == "Deployment" and $names | contains(.metadata.name)) | del(.)' *.yaml

yq-v4

https://sockstack.cn/reproduce/ORXRyZYNRRiqL45

作者

moon

许可协议

CC BY 4.0

发布于

2023-08-29

修改于

2025-02-03

上一篇：轻量级数据中台，大中型企业数字化转型首选下一篇：公网远程访问局域网SQL Server数据库

yq-v4

一. 安装

二. yaml 案例

1. coco.yaml

2. 1-backup.yaml

3. pod.yaml

三. 使用案例

1. 标量类型查询

2. Map-Array类型

3. 变量引用

4. 增，删，改

1). 增

2）改

3). 删除

文章分类

博客重构之路

Spring Boot简单入门

k8s 入门教程

MySQL 知识

NSQ 消息队列

ThinkPHP5 源码分析

使用 Docker 从零开始搭建私人代码仓库

日常开发汇总

标签列表

springboot

hyperf

swoole

webman

php

多线程

数据结构

docker

k8s

thinkphp

mysql

tailwindcss

flowbite

css

前端