AEWS Study #7 – EKS Automation

# 서비스명 변수 지정
export SERVICE=s3

# helm 차트 다운로드
export RELEASE_VERSION=$(curl -sL https://api.github.com/repos/aws-controllers-k8s/$SERVICE-controller/releases/latest | grep '"tag_name":' | cut -d'"' -f4 | cut -c 2-)
helm pull oci://public.ecr.aws/aws-controllers-k8s/$SERVICE-chart --version=$RELEASE_VERSION
tar xzvf $SERVICE-chart-$RELEASE_VERSION.tgz

# helm chart 확인
tree ~/$SERVICE-chart

# ACK S3 Controller 설치
export ACK_SYSTEM_NAMESPACE=ack-system
export AWS_REGION=ap-northeast-2
helm install --create-namespace -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller --set aws.region="$AWS_REGION" ~/$SERVICE-chart

# 설치 확인
helm list --namespace $ACK_SYSTEM_NAMESPACE
NAME             	NAMESPACE 	REVISION	UPDATED                                	STATUS  	CHART         	APP VERSION
ack-s3-controller	ack-system	1       	2023-06-04 21:12:06.857876402 +0900 KST	deployed	s3-chart-1.0.4	1.0.4

kubectl -n ack-system get pods
NAME                                          READY   STATUS    RESTARTS   AGE
ack-s3-controller-s3-chart-7c55c6657d-mbrq5   1/1     Running   0          10s

kubectl get crd | grep $SERVICE
buckets.s3.services.k8s.aws                  2023-06-04T12:12:04Z

kubectl get all -n ack-system
NAME                                              READY   STATUS    RESTARTS   AGE
pod/ack-s3-controller-s3-chart-7c55c6657d-mbrq5   1/1     Running   0          30s
NAME                                         READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/ack-s3-controller-s3-chart   1/1     1            1           30s
NAME                                                    DESIRED   CURRENT   READY   AGE
replicaset.apps/ack-s3-controller-s3-chart-7c55c6657d   1         1         1       30s

kubectl get-all -n ack-system
kubectl describe sa -n ack-system ack-s3-controller
Name:                ack-s3-controller
Namespace:           ack-system
Labels:              app.kubernetes.io/instance=ack-s3-controller
                     app.kubernetes.io/managed-by=Helm
....

# Create an iamserviceaccount - AWS IAM role bound to a Kubernetes service account
eksctl create iamserviceaccount \
  --name ack-$SERVICE-controller \
  --namespace ack-system \
  --cluster $CLUSTER_NAME \
  --attach-policy-arn $(aws iam list-policies --query 'Policies[?PolicyName==`AmazonS3FullAccess`].Arn' --output text) \
  --override-existing-serviceaccounts --approve

# 확인 >> 웹 관리 콘솔에서 CloudFormation Stack >> IAM Role 확인
eksctl get iamserviceaccount --cluster $CLUSTER_NAME
NAMESPACE	NAME				ROLE ARN
ack-system	ack-s3-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-XFQ0LIYFYJ3Z

# Inspecting the newly created Kubernetes Service Account, we can see the role we want it to assume in our pod.
kubectl get sa -n ack-system
NAME                SECRETS   AGE
ack-s3-controller   0         5m16s

kubectl describe sa ack-$SERVICE-controller -n ack-system
Name:                ack-s3-controller
Namespace:           ack-system
Labels:              app.kubernetes.io/instance=ack-s3-controller
                     app.kubernetes.io/managed-by=eksctl
...

# Restart ACK service controller deployment using the following commands.
kubectl -n ack-system rollout restart deploy ack-$SERVICE-controller-$SERVICE-chart

# IRSA 적용으로 Env, Volume 추가 확인
kubectl describe pod -n ack-system -l k8s-app=$SERVICE-chart
Name:             ack-s3-controller-s3-chart-559866764-42xt6
Namespace:        ack-system
Priority:         0
Service Account:  ack-s3-controller
Node:             ip-192-168-1-10.ap-northeast-2.compute.internal/192.168.1.10
...

# [터미널1] 모니터링
watch -d aws s3 ls

# S3 버킷 생성을 위한 설정 파일 생성
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text)
export BUCKET_NAME=my-ack-s3-bucket-$AWS_ACCOUNT_ID

read -r -d '' BUCKET_MANIFEST <<EOF
apiVersion: s3.services.k8s.aws/v1alpha1
kind: Bucket
metadata:
  name: $BUCKET_NAME
spec:
  name: $BUCKET_NAME
EOF

echo "${BUCKET_MANIFEST}" > bucket.yaml
cat bucket.yaml | yh
apiVersion: s3.services.k8s.aws/v1alpha1
kind: Bucket
metadata:
  name: my-ack-s3-bucket-MyAccount
spec:
  name: my-ack-s3-bucket-MyAccount

# S3 버킷 생성
aws s3 ls
kubectl create -f bucket.yaml
bucket.s3.services.k8s.aws/my-ack-s3-bucket-MyAccount created

# S3 버킷 확인
aws s3 ls
2023-06-04 21:21:43 my-ack-s3-bucket-MyAccount

kubectl get buckets
NAME                            AGE
my-ack-s3-bucket-MyAccount   17s

kubectl describe bucket/$BUCKET_NAME | head -6
Name:         my-ack-s3-bucket-MyAccount
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  s3.services.k8s.aws/v1alpha1
Kind:         Bucket

# S3 버킷 업데이트 : 태그 정보 입력
read -r -d '' BUCKET_MANIFEST <<EOF
apiVersion: s3.services.k8s.aws/v1alpha1
kind: Bucket
metadata:
  name: $BUCKET_NAME
spec:
  name: $BUCKET_NAME
  tagging:
    tagSet:
    - key: myTagKey
      value: myTagValue
EOF

echo "${BUCKET_MANIFEST}" > bucket.yaml

# 변경 내용 사전 확인
cat bucket.yaml | yh
apiVersion: s3.services.k8s.aws/v1alpha1
kind: Bucket
metadata:
  name: my-ack-s3-bucket-587122150371
spec:
  name: my-ack-s3-bucket-587122150371
  tagging:
    tagSet:
    - key: myTagKey
      value: myTagValue

# S3 버킷 설정 업데이트 실행 : 필요 주석 자동 업뎃 내용이니 무시해도됨!
kubectl apply -f bucket.yaml

# S3 버킷 업데이트 확인 
kubectl describe bucket/$BUCKET_NAME | grep Spec: -A5
Spec:
  Name:  my-ack-s3-bucket-MyAccount
  Tagging:
    Tag Set:
      Key:    myTagKey
      Value:  myTagValue

# S3 버킷 삭제
kubectl delete -f bucket.yaml

# verify the bucket no longer exists
kubectl get bucket/$BUCKET_NAME
aws s3 ls | grep $BUCKET_NAME

# helm uninstall
export SERVICE=s3
helm uninstall -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller

# ACK S3 Controller 관련 crd 삭제
kubectl delete -f ~/$SERVICE-chart/crds

# IRSA 삭제
eksctl delete iamserviceaccount --cluster myeks --name ack-$SERVICE-controller --namespace ack-system

##### namespace 삭제 >> ACK 모든 실습 후 삭제  #####
kubectl delete namespace $ACK_K8S_NAMESPACE

# 서비스명 변수 지정 및 helm 차트 다운로드
export SERVICE=ec2
export RELEASE_VERSION=$(curl -sL https://api.github.com/repos/aws-controllers-k8s/$SERVICE-controller/releases/latest | grep '"tag_name":' | cut -d'"' -f4 | cut -c 2-)
helm pull oci://public.ecr.aws/aws-controllers-k8s/$SERVICE-chart --version=$RELEASE_VERSION
tar xzvf $SERVICE-chart-$RELEASE_VERSION.tgz

# helm chart 확인
tree ~/$SERVICE-chart

# ACK EC2-Controller 설치
export ACK_SYSTEM_NAMESPACE=ack-system
export AWS_REGION=ap-northeast-2
helm install -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller --set aws.region="$AWS_REGION" ~/$SERVICE-chart
NAME: ack-ec2-controller
LAST DEPLOYED: Sun Jun  4 21:46:39 2023
NAMESPACE: ack-system
STATUS: deployed

# 설치 확인
helm list --namespace $ACK_SYSTEM_NAMESPACE
NAME              	NAMESPACE 	REVISION	UPDATED                               	STATUS  	CHART          	APP VERSION
ack-ec2-controller	ack-system	1       	2023-06-04 21:46:39.80730411 +0900 KST	deployed	ec2-chart-1.0.3	1.0.3

kubectl -n $ACK_SYSTEM_NAMESPACE get pods -l "app.kubernetes.io/instance=ack-$SERVICE-controller"
NAME                                            READY   STATUS    RESTARTS   AGE
ack-ec2-controller-ec2-chart-777567ff4c-45s2d   1/1     Running   0          21s

kubectl get crd | grep $SERVICE
dhcpoptions.ec2.services.k8s.aws             2023-06-04T12:46:38Z
elasticipaddresses.ec2.services.k8s.aws      2023-06-04T12:46:38Z
instances.ec2.services.k8s.aws               2023-06-04T12:46:38Z
internetgateways.ec2.services.k8s.aws        2023-06-04T12:46:38Z
natgateways.ec2.services.k8s.aws             2023-06-04T12:46:39Z
routetables.ec2.services.k8s.aws             2023-06-04T12:46:39Z
securitygroups.ec2.services.k8s.aws          2023-06-04T12:46:39Z
subnets.ec2.services.k8s.aws                 2023-06-04T12:46:39Z
transitgateways.ec2.services.k8s.aws         2023-06-04T12:46:39Z
vpcendpoints.ec2.services.k8s.aws            2023-06-04T12:46:39Z
vpcs.ec2.services.k8s.aws                    2023-06-04T12:46:39Z

# Create an iamserviceaccount - AWS IAM role bound to a Kubernetes service account
eksctl create iamserviceaccount \
  --name ack-$SERVICE-controller \
  --namespace $ACK_SYSTEM_NAMESPACE \
  --cluster $CLUSTER_NAME \
  --attach-policy-arn $(aws iam list-policies --query 'Policies[?PolicyName==`AmazonEC2FullAccess`].Arn' --output text) \
  --override-existing-serviceaccounts --approve

# 확인 >> 웹 관리 콘솔에서 CloudFormation Stack >> IAM Role 확인
eksctl get iamserviceaccount --cluster $CLUSTER_NAME
NAMESPACE	NAME				ROLE ARN
ack-system	ack-ec2-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-VK23PW7Y288N

# Inspecting the newly created Kubernetes Service Account, we can see the role we want it to assume in our pod.
kubectl get sa -n $ACK_SYSTEM_NAMESPACE
NAME                 SECRETS   AGE
ack-ec2-controller   0         3m45s

kubectl describe sa ack-$SERVICE-controller -n $ACK_SYSTEM_NAMESPACE
Name:                ack-ec2-controller
Namespace:           ack-system
Labels:              app.kubernetes.io/instance=ack-ec2-controller
                     app.kubernetes.io/managed-by=eksctl
                     app.kubernetes.io/name=ec2-chart
...

# Restart ACK service controller deployment using the following commands.
kubectl -n $ACK_SYSTEM_NAMESPACE rollout restart deploy ack-$SERVICE-controller-$SERVICE-chart
deployment.apps/ack-ec2-controller-ec2-chart restarted

# IRSA 적용으로 Env, Volume 추가 확인
kubectl describe pod -n $ACK_SYSTEM_NAMESPACE -l k8s-app=$SERVICE-chart
Name:             ack-ec2-controller-ec2-chart-76dd69c88-62kfs
Namespace:        ack-system
Priority:         0
Service Account:  ack-ec2-controller
Node:             ip-192-168-1-10.ap-northeast-2.compute.internal/192.168.1.10
...

# [터미널1] 모니터링
while true; do aws ec2 describe-vpcs --query 'Vpcs[*].{VPCId:VpcId, CidrBlock:CidrBlock}' --output text; echo "-----"; sleep 1; done
-----
192.168.0.0/16	vpc-0ab487a4560d0e009
172.31.0.0/16	vpc-04663e26208f8fb7d
-----

# VPC 생성
cat <<EOF > vpc.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: VPC
metadata:
  name: vpc-tutorial-test
spec:
  cidrBlocks: 
  - 10.0.0.0/16
  enableDNSSupport: true
  enableDNSHostnames: true
EOF
 
kubectl apply -f vpc.yaml
vpc.ec2.services.k8s.aws/vpc-tutorial-test created

-----
192.168.0.0/16	vpc-0ab487a4560d0e009
172.31.0.0/16	vpc-04663e26208f8fb7d
10.0.0.0/16	vpc-0cc88d9792531d28b
-----

# VPC 생성 확인
kubectl get vpcs
NAME                ID                      STATE
vpc-tutorial-test   vpc-0cc88d9792531d28b   available

kubectl describe vpcs
Name:         vpc-tutorial-test
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  ec2.services.k8s.aws/v1alpha1
...

aws ec2 describe-vpcs --query 'Vpcs[*].{VPCId:VpcId, CidrBlock:CidrBlock}' --output text
192.168.0.0/16	vpc-0ab487a4560d0e009
172.31.0.0/16	vpc-04663e26208f8fb7d
10.0.0.0/16	vpc-0cc88d9792531d28b

# [터미널1] 모니터링
VPCID=$(kubectl get vpcs vpc-tutorial-test -o jsonpath={.status.vpcID})
while true; do aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPCID" --query 'Subnets[*].{SubnetId:SubnetId, CidrBlock:CidrBlock}' --output text; echo "-----"; sleep 1 ; done
-----
-----

# 서브넷 생성
VPCID=$(kubectl get vpcs vpc-tutorial-test -o jsonpath={.status.vpcID})

cat <<EOF > subnet.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: Subnet
metadata:
  name: subnet-tutorial-test
spec:
  cidrBlock: 10.0.0.0/20
  vpcID: $VPCID
EOF
kubectl apply -f subnet.yaml
-----
10.0.0.0/20	subnet-03d5af788ba04433d
-----

# 서브넷 생성 확인
kubectl get subnets
kubectl describe subnets
aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPCID" --query 'Subnets[*].{SubnetId:SubnetId, CidrBlock:CidrBlock}' --output text

# 리소스 삭제
kubectl delete -f subnet.yaml && kubectl delete -f vpc.yaml

cat <<EOF > vpc-workflow.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: VPC
metadata:
  name: tutorial-vpc
spec:
  cidrBlocks: 
  - 10.0.0.0/16
  enableDNSSupport: true
  enableDNSHostnames: true
  tags:
    - key: name
      value: vpc-tutorial
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: InternetGateway
metadata:
  name: tutorial-igw
spec:
  vpcRef:
    from:
      name: tutorial-vpc
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: NATGateway
metadata:
  name: tutorial-natgateway1
spec:
  subnetRef:
    from:
      name: tutorial-public-subnet1
  allocationRef:
    from:
      name: tutorial-eip1
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: ElasticIPAddress
metadata:
  name: tutorial-eip1
spec:
  tags:
    - key: name
      value: eip-tutorial
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: RouteTable
metadata:
  name: tutorial-public-route-table
spec:
  vpcRef:
    from:
      name: tutorial-vpc
  routes:
  - destinationCIDRBlock: 0.0.0.0/0
    gatewayRef:
      from:
        name: tutorial-igw
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: RouteTable
metadata:
  name: tutorial-private-route-table-az1
spec:
  vpcRef:
    from:
      name: tutorial-vpc
  routes:
  - destinationCIDRBlock: 0.0.0.0/0
    natGatewayRef:
      from:
        name: tutorial-natgateway1
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: Subnet
metadata:
  name: tutorial-public-subnet1
spec:
  availabilityZone: ap-northeast-2a
  cidrBlock: 10.0.0.0/20
  mapPublicIPOnLaunch: true
  vpcRef:
    from:
      name: tutorial-vpc
  routeTableRefs:
  - from:
      name: tutorial-public-route-table
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: Subnet
metadata:
  name: tutorial-private-subnet1
spec:
  availabilityZone: ap-northeast-2a
  cidrBlock: 10.0.128.0/20
  vpcRef:
    from:
      name: tutorial-vpc
  routeTableRefs:
  - from:
      name: tutorial-private-route-table-az1
---
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: SecurityGroup
metadata:
  name: tutorial-security-group
spec:
  description: "ack security group"
  name: tutorial-sg
  vpcRef:
     from:
       name: tutorial-vpc
  ingressRules:
    - ipProtocol: tcp
      fromPort: 22
      toPort: 22
      ipRanges:
        - cidrIP: "0.0.0.0/0"
          description: "ingress"
EOF

# VPC 환경 생성
kubectl apply -f vpc-workflow.yaml

# [터미널1] NATGW 생성 완료 후 tutorial-private-route-table-az1 라우팅 테이블 ID가 확인되고 그후 tutorial-private-subnet1 서브넷ID가 확인됨 > 5분 정도 시간 소요
watch -d kubectl get routetables,subnet


# VPC 환경 생성 확인
kubectl describe vpcs
Name:         tutorial-vpc
Namespace:    default
...
kubectl describe internetgateways
Name:         tutorial-igw
Namespace:    default
...
kubectl describe routetables
Name:         tutorial-private-route-table-az1
Namespace:    default
...
kubectl describe natgateways
Name:         tutorial-natgateway1
Namespace:    default
...
kubectl describe elasticipaddresses
Name:         tutorial-eip1
Namespace:    default
...
kubectl describe securitygroups
Name:         tutorial-security-group
Namespace:    default
...

# 배포 도중 2개의 서브넷 상태 정보 비교 해보자
kubectl describe subnets
...
Status:
  Conditions:
    Last Transition Time:  2023-06-04T13:03:04Z
    Message:               Reference resolution failed
    Reason:                the referenced resource is not synced yet. resource:RouteTable, namespace:default, name:tutorial-private-route-table-az1
    Status:                Unknown
    Type:                  ACK.ReferencesResolved
...
Status:
  Ack Resource Metadata:
    Arn:                       arn:aws:ec2:ap-northeast-2:MyAccount:subnet/subnet-03cd0923f8663f1ee
    Owner Account ID:          MyAccount
    Region:                    ap-northeast-2
  Available IP Address Count:  4091
  Conditions:
    Last Transition Time:           2023-06-04T13:01:45Z
    Status:                         True
    Type:                           ACK.ReferencesResolved
    Last Transition Time:           2023-06-04T13:01:45Z
    Message:                        Resource synced successfully
    Reason:
    Status:                         True
    Type:                           ACK.ResourceSynced
...

# public 서브넷 ID 확인
PUBSUB1=$(kubectl get subnets tutorial-public-subnet1 -o jsonpath={.status.subnetID})
echo $PUBSUB1
subnet-03cd0923f8663f1ee

# 보안그룹 ID 확인
TSG=$(kubectl get securitygroups tutorial-security-group -o jsonpath={.status.id})
echo $TSG
sg-0992f933b69408947

# Amazon Linux 2 최신 AMI ID 확인
AL2AMI=$(aws ec2 describe-images --owners amazon --filters "Name=name,Values=amzn2-ami-hvm-2.0.*-x86_64-gp2" --query 'Images[0].ImageId' --output text)
echo $AL2AMI
ami-0a0453c1a1758acf1

# 각자 자신의 SSH 키페어 이름 변수 지정
MYKEYPAIR=<각자 자신의 SSH 키페어 이름>
MYKEYPAIR=aewspair

# 변수 확인 > 특히 서브넷 ID가 확인되었는지 꼭 확인하자!
echo $PUBSUB1 , $TSG , $AL2AMI , $MYKEYPAIR
subnet-03cd0923f8663f1ee , sg-0992f933b69408947 , ami-0a0453c1a1758acf1 , aewspair

# [터미널1] 모니터링
while true; do aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,PrivateIPAdd:PrivateIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output table; date ; sleep 1 ; done
+----------------+-----------------+------------------+----------+
|  InstanceName  |  PrivateIPAdd   |   PublicIPAdd    | Status   |
+----------------+-----------------+------------------+----------+
|  myeks-ng1-Node|  192.168.3.165  |  54.180.xxx.xxx  |  running |
|  myeks-ng1-Node|  192.168.2.5    |  3.35.xxx.xxx    |  running |
|  myeks-ng1-Node|  192.168.1.10   |  43.201.xxx.xxx  |  running |
|  myeks-bastion |  192.168.1.100  |  54.180.xxx.xxx  |  running |
+----------------+-----------------+------------------+----------+
Sun Jun  4 22:09:42 KST 2023

# public 서브넷에 인스턴스 생성
cat <<EOF > tutorial-bastion-host.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: Instance
metadata:
  name: tutorial-bastion-host
spec:
  imageID: $AL2AMI # AL2 AMI ID - ap-northeast-2
  instanceType: t3.medium
  subnetID: $PUBSUB1
  securityGroupIDs:
  - $TSG
  keyName: $MYKEYPAIR
  tags:
    - key: producer
      value: ack
EOF
kubectl apply -f tutorial-bastion-host.yaml
instance.ec2.services.k8s.aws/tutorial-bastion-host created


# 인스턴스 생성 확인
kubectl get instance
NAME                    ID
tutorial-bastion-host   i-070849e21be1aa100

kubectl describe instance
Name:         tutorial-bastion-host
Namespace:    default

aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,PrivateIPAdd:PrivateIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output table
+----------------+-----------------+------------------+----------+
|  InstanceName  |  PrivateIPAdd   |   PublicIPAdd    | Status   |
+----------------+-----------------+------------------+----------+
|  myeks-ng1-Node|  192.168.3.165  |  54.180.xxx.xxx  |  running |
|  myeks-ng1-Node|  192.168.2.5    |  3.35.xxx.xxx    |  running |
|  myeks-ng1-Node|  192.168.1.10   |  43.201.xxx.xxx  |  running |
|  myeks-bastion |  192.168.1.100  |  54.180.xxx.xxx  |  running |
|  None          |  10.0.5.137     |  13.209.xxx.xxx  |  running |
+----------------+-----------------+------------------+----------+
Sun Jun  4 22:10:42 KST 2023

ssh -i <자신의 키페어파일> ec2-user@<public 서브넷에 인스턴스 퍼블릭IP>
------
# public 서브넷에 인스턴스 접속 후 외부 인터넷 통신 여부 확인 
ping -c 2 8.8.8.8
exit
------

# SG egress 설정
cat <<EOF > modify-sg.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: SecurityGroup
metadata:
  name: tutorial-security-group
spec:
  description: "ack security group"
  name: tutorial-sg
  vpcRef:
     from:
       name: tutorial-vpc
  ingressRules:
    - ipProtocol: tcp
      fromPort: 22
      toPort: 22
      ipRanges:
        - cidrIP: "0.0.0.0/0"
          description: "ingress"
  egressRules:
    - ipProtocol: '-1'
      ipRanges:
        - cidrIP: "0.0.0.0/0"
          description: "egress"
EOF
kubectl apply -f modify-sg.yaml
securitygroup.ec2.services.k8s.aws/tutorial-security-group configured

# 변경 확인 >> 보안그룹에 아웃바운드 규칙 확인
kubectl logs -n $ACK_SYSTEM_NAMESPACE -l k8s-app=ec2-chart -f
2023-06-04T13:16:36.970Z	INFO	ackrt	desired resource state has changed	{"account": "MyAccount", "role": "", "region": "ap-northeast-2", "kind": "SecurityGroup", "namespace": "default", "name": "tutorial-security-group", "is_adopted": false, "generation": 2, "diff": [{"Path":{"Parts":["Spec","EgressRules"]},"A":[{"ipProtocol":"-1","ipRanges":[{"cidrIP":"0.0.0.0/0","description":"egress"}]}],"B":null}]}
2023-06-04T13:16:37.217Z	INFO	ackrt	updated resource	{"account": "MyAccount", "role": "", "region": "ap-northeast-2", "kind": "SecurityGroup", "namespace": "default", "name": "tutorial-security-group", "is_adopted": false, "generation": 2}

ssh -i <자신의 키페어파일> ec2-user@<public 서브넷에 인스턴스 퍼블릭IP>
------
# public 서브넷에 인스턴스 접속 후 외부 인터넷 통신 여부 확인 
ping -c 10 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
64 bytes from 8.8.8.8: icmp_seq=1 ttl=46 time=17.0 ms
64 bytes from 8.8.8.8: icmp_seq=2 ttl=46 time=17.1 ms
64 bytes from 8.8.8.8: icmp_seq=3 ttl=46 time=17.1 ms
64 bytes from 8.8.8.8: icmp_seq=4 ttl=46 time=17.1 ms
...
--- 8.8.8.8 ping statistics ---
10 packets transmitted, 10 received, 0% packet loss, time 9011ms
rtt min/avg/max/mdev = 17.089/17.135/17.164/0.156 ms

curl ipinfo.io/ip ; echo
13.209.xxx.xxx
exit
------

# private 서브넷 ID 확인
PRISUB1=$(kubectl get subnets tutorial-private-subnet1 -o jsonpath={.status.subnetID})
echo $PRISUB1
subnet-0997be6d046a7f8c7

# 변수 확인 > 특히 private 서브넷 ID가 확인되었는지 꼭 확인하자!
echo $PRISUB1 , $TSG , $AL2AMI , $MYKEYPAIR
subnet-0997be6d046a7f8c7 , sg-0992f933b69408947 , ami-0a0453c1a1758acf1 , aewspair

# private 서브넷에 인스턴스 생성
cat <<EOF > tutorial-instance-private.yaml
apiVersion: ec2.services.k8s.aws/v1alpha1
kind: Instance
metadata:
  name: tutorial-instance-private
spec:
  imageID: $AL2AMI # AL2 AMI ID - ap-northeast-2
  instanceType: t3.medium
  subnetID: $PRISUB1
  securityGroupIDs:
  - $TSG
  keyName: $MYKEYPAIR
  tags:
    - key: producer
      value: ack
EOF
kubectl apply -f tutorial-instance-private.yaml
instance.ec2.services.k8s.aws/tutorial-instance-private created

# 인스턴스 생성 확인 (위에서 만든 Public Subnet EC2는 bastion, 방금 만든 EC2는 instance-private)
kubectl get instance
NAME                        ID
tutorial-bastion-host       i-070849e21be1aa100
tutorial-instance-private   i-088273052920e222b

kubectl describe instance
Name:         tutorial-instance-private
Namespace:    default

aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,PrivateIPAdd:PrivateIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output table
+----------------+-----------------+------------------+----------+
|  InstanceName  |  PrivateIPAdd   |   PublicIPAdd    | Status   |
+----------------+-----------------+------------------+----------+
|  myeks-ng1-Node|  192.168.3.165  |  54.180.xxx.xxx  |  running |
|  myeks-ng1-Node|  192.168.2.5    |  3.35.xxx.xxx    |  running |
|  myeks-ng1-Node|  192.168.1.10   |  43.201.xxx.xxx  |  running |
|  myeks-bastion |  192.168.1.100  |  54.180.xxx.xxx  |  running |
|  None          |  10.0.5.137     |  13.209.xxx.xxx  |  running |
|  None          |  10.0.128.130   |  None            |  running |
+----------------+-----------------+------------------+----------+

# [터미널3] SSH 터널링 설정
ssh -i <자신의 키페어파일> -L <자신의 임의 로컬 포트>:<private 서브넷의 인스턴스의 private ip 주소>:22 ec2-user@<public 서브넷에 인스턴스 퍼블릭IP> -v
ssh -i aewspair.pem -L 9999:10.0.128.130:22 ec2-user@13.209.xxx.xxx -v
---

# [터미널4] SSH 터널링 통해 Private Subnet EC2 접속
ssh -i <자신의 키페어파일> -p <자신의 임의 로컬 포트> ec2-user@localhost
ssh -i aewspair.pem -p 9999 ec2-user@localhost
---
# IP 및 네트워크 정보 확인
ip -c addr
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9001 qdisc mq state UP group default qlen 1000
    link/ether 02:93:25:a2:64:64 brd ff:ff:ff:ff:ff:ff
    inet 10.0.128.130/20 brd 10.0.143.255 scope global dynamic eth0

sudo ss -tnp
State       Recv-Q       Send-Q              Local Address:Port                Peer Address:Port        Process
ESTAB       0            0                    10.0.128.130:22                    10.0.5.137:38416        users:(("sshd",pid=2516,fd=3),("sshd",pid=2499,fd=3))

ping -c 2 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
64 bytes from 8.8.8.8: icmp_seq=1 ttl=103 time=28.7 ms
64 bytes from 8.8.8.8: icmp_seq=2 ttl=103 time=27.9 ms
--- 8.8.8.8 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1001ms
rtt min/avg/max/mdev = 27.994/28.396/28.799/0.436 ms

curl ipinfo.io/ip ; echo 
43.201.xxx.xxx
exit
---

# 리소스 삭제
kubectl delete -f tutorial-bastion-host.yaml && kubectl delete -f tutorial-instance-private.yaml
kubectl delete -f vpc-workflow.yaml  # vpc 관련 모든 리소스들 삭제에는 다소 시간이 소요됨

# 리소스 삭제 확인
kubectl get instance
No resources found in default namespace.

aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,PrivateIPAdd:PrivateIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output table
+----------------+-----------------+------------------+----------+
|  InstanceName  |  PrivateIPAdd   |   PublicIPAdd    | Status   |
+----------------+-----------------+------------------+----------+
|  myeks-ng1-Node|  192.168.3.165  |  54.180.xxx.xxx  |  running |
|  myeks-ng1-Node|  192.168.2.5    |  3.35.xxx.xxx    |  running |
|  myeks-ng1-Node|  192.168.1.10   |  43.201.xxx.xxx  |  running |
|  myeks-bastion |  192.168.1.100  |  54.180.xxx.xxx  |  running |
+----------------+-----------------+------------------+----------+

# 서비스명 변수 지정 및 helm 차트 다운로드
export SERVICE=rds
export RELEASE_VERSION=$(curl -sL https://api.github.com/repos/aws-controllers-k8s/$SERVICE-controller/releases/latest | grep '"tag_name":' | cut -d'"' -f4 | cut -c 2-)
helm pull oci://public.ecr.aws/aws-controllers-k8s/$SERVICE-chart --version=$RELEASE_VERSION
tar xzvf $SERVICE-chart-$RELEASE_VERSION.tgz

# helm chart 확인
tree ~/$SERVICE-chart

# ACK RDS-Controller 설치
export ACK_SYSTEM_NAMESPACE=ack-system
export AWS_REGION=ap-northeast-2
helm install -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller --set aws.region="$AWS_REGION" ~/$SERVICE-chart

# 설치 확인
helm list --namespace $ACK_SYSTEM_NAMESPACE
NAME              	NAMESPACE 	REVISION	UPDATED                                	STATUS  	CHART          	APP VERSION
ack-ec2-controller	ack-system	1       	2023-06-04 21:46:39.80730411 +0900 KST 	deployed	ec2-chart-1.0.3	1.0.3
ack-rds-controller	ack-system	1       	2023-06-05 08:24:20.821985798 +0900 KST	deployed	rds-chart-1.1.4	1.1.4

kubectl -n $ACK_SYSTEM_NAMESPACE get pods -l "app.kubernetes.io/instance=ack-$SERVICE-controller"
NAME                                            READY   STATUS              RESTARTS   AGE
ack-rds-controller-rds-chart-6d59dfdfd7-k2mnl   0/1     ContainerCreating   0          2s

kubectl get crd | grep $SERVICE
dbclusterparametergroups.rds.services.k8s.aws   2023-06-04T23:24:19Z
dbclusters.rds.services.k8s.aws                 2023-06-04T23:24:19Z
dbinstances.rds.services.k8s.aws                2023-06-04T23:24:20Z
dbparametergroups.rds.services.k8s.aws          2023-06-04T23:24:20Z
dbproxies.rds.services.k8s.aws                  2023-06-04T23:24:20Z
dbsubnetgroups.rds.services.k8s.aws             2023-06-04T23:24:20Z
globalclusters.rds.services.k8s.aws             2023-06-04T23:24:20Z

# Create an iamserviceaccount - AWS IAM role bound to a Kubernetes service account
eksctl create iamserviceaccount \
  --name ack-$SERVICE-controller \
  --namespace $ACK_SYSTEM_NAMESPACE \
  --cluster $CLUSTER_NAME \
  --attach-policy-arn $(aws iam list-policies --query 'Policies[?PolicyName==`AmazonRDSFullAccess`].Arn' --output text) \
  --override-existing-serviceaccounts --approve

# 확인 >> 웹 관리 콘솔에서 CloudFormation Stack >> IAM Role 확인
eksctl get iamserviceaccount --cluster $CLUSTER_NAME
NAMESPACE	NAME				ROLE ARN
ack-system	ack-ec2-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-VK23PW7Y288N
ack-system	ack-rds-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-AUTL0REK98XM
ack-system	ack-s3-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-XFQ0LIYFYJ3Z

# Inspecting the newly created Kubernetes Service Account, we can see the role we want it to assume in our pod.
kubectl get sa -n $ACK_SYSTEM_NAMESPACE
NAME                 SECRETS   AGE
ack-ec2-controller   0         10h
ack-rds-controller   0         5m2s

kubectl describe sa ack-$SERVICE-controller -n $ACK_SYSTEM_NAMESPACE
Name:                ack-rds-controller
Namespace:           ack-system
Labels:              app.kubernetes.io/instance=ack-rds-controller
                     app.kubernetes.io/managed-by=eksctl
                     app.kubernetes.io/name=rds-chart

# Restart ACK service controller deployment using the following commands.
kubectl -n $ACK_SYSTEM_NAMESPACE rollout restart deploy ack-$SERVICE-controller-$SERVICE-chart
deployment.apps/ack-rds-controller-rds-chart restarted

# IRSA 적용으로 Env, Volume 추가 확인
kubectl describe pod -n $ACK_SYSTEM_NAMESPACE -l k8s-app=$SERVICE-chart
Name:             ack-rds-controller-rds-chart-5dfbf7dccb-dlwdv
Namespace:        ack-system
Priority:         0
Service Account:  ack-rds-controller
Node:             ip-192-168-2-5.ap-northeast-2.compute.internal/192.168.2.5
...

# DB 암호를 위한 secret 생성
RDS_INSTANCE_NAME="<your instance name>"
RDS_INSTANCE_PASSWORD="<your instance password>"
RDS_INSTANCE_NAME=myrds
RDS_INSTANCE_PASSWORD=qwe12345
kubectl create secret generic "${RDS_INSTANCE_NAME}-password" --from-literal=password="${RDS_INSTANCE_PASSWORD}"
secret/myrds-password created

# 확인
kubectl get secret $RDS_INSTANCE_NAME-password
NAME             TYPE     DATA   AGE
myrds-password   Opaque   1      10s

# [터미널1] 모니터링
RDS_INSTANCE_NAME=myrds
watch -d "kubectl describe dbinstance "${RDS_INSTANCE_NAME}" | grep 'Db Instance Status'"

# RDS 배포 생성 : 15분 이내 시간 소요 >> 보안그룹, 서브넷 등 필요한 옵션들은 추가해서 설정해보자!
cat <<EOF > rds-mariadb.yaml
apiVersion: rds.services.k8s.aws/v1alpha1
kind: DBInstance
metadata:
  name: "${RDS_INSTANCE_NAME}"
spec:
  allocatedStorage: 20
  dbInstanceClass: db.t4g.micro
  dbInstanceIdentifier: "${RDS_INSTANCE_NAME}"
  engine: mariadb
  engineVersion: "10.6"
  masterUsername: "admin"
  masterUserPassword:
    namespace: default
    name: "${RDS_INSTANCE_NAME}-password"
    key: password
EOF
kubectl apply -f rds-mariadb.yaml

# 생성 확인
kubectl get dbinstances  ${RDS_INSTANCE_NAME}
NAME    STATUS
myrds   creating

kubectl describe dbinstance "${RDS_INSTANCE_NAME}"
Name:         myrds
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  rds.services.k8s.aws/v1alpha1
...

aws rds describe-db-instances --db-instance-identifier $RDS_INSTANCE_NAME | jq
  Db Instance Status:         available

# 생성 완료 대기 : for 지정 상태가 완료되면 정상 종료됨
kubectl wait dbinstances ${RDS_INSTANCE_NAME} --for=condition=ACK.ResourceSynced --timeout=15m
dbinstance.rds.services.k8s.aws/myrds condition met

# Configmap 구성 전 Configmap 확인
kubectl get configmaps
NAME               DATA   AGE
kube-root-ca.crt   1      12h

# Configmap 구성
RDS_INSTANCE_CONN_CM="${RDS_INSTANCE_NAME}-conn-cm"

cat <<EOF > rds-field-exports.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: ${RDS_INSTANCE_CONN_CM}
data: {}
---
apiVersion: services.k8s.aws/v1alpha1
kind: FieldExport
metadata:
  name: ${RDS_INSTANCE_NAME}-host
spec:
  to:
    name: ${RDS_INSTANCE_CONN_CM}
    kind: configmap
  from:
    path: ".status.endpoint.address"
    resource:
      group: rds.services.k8s.aws
      kind: DBInstance
      name: ${RDS_INSTANCE_NAME}
---
apiVersion: services.k8s.aws/v1alpha1
kind: FieldExport
metadata:
  name: ${RDS_INSTANCE_NAME}-port
spec:
  to:
    name: ${RDS_INSTANCE_CONN_CM}
    kind: configmap
  from:
    path: ".status.endpoint.port"
    resource:
      group: rds.services.k8s.aws
      kind: DBInstance
      name: ${RDS_INSTANCE_NAME}
---
apiVersion: services.k8s.aws/v1alpha1
kind: FieldExport
metadata:
  name: ${RDS_INSTANCE_NAME}-user
spec:
  to:
    name: ${RDS_INSTANCE_CONN_CM}
    kind: configmap
  from:
    path: ".spec.masterUsername"
    resource:
      group: rds.services.k8s.aws
      kind: DBInstance
      name: ${RDS_INSTANCE_NAME}
EOF

kubectl apply -f rds-field-exports.yaml

# Configmap 구성 확인
NAME               DATA   AGE
kube-root-ca.crt   1      12h
myrds-conn-cm      3      2m35s

# 상태 정보 확인 : address 와 port 정보 
kubectl get dbinstances myrds -o jsonpath={.status.endpoint} | jq
{
  "address": "myrds.cnlc7l....ap-northeast-2.rds.amazonaws.com",
  "hostedZoneID": "ZLA2NUC...",
  "port": 3306
}

# 상태 정보 확인 : masterUsername 확인
kubectl get dbinstances myrds -o jsonpath={.spec.masterUsername} ; echo
admin

# 컨피그맵 확인
kubectl get cm myrds-conn-cm -o yaml | kubectl neat | yh
apiVersion: v1
data:
  default.myrds-host: myrds.cnlc7....ap-northeast-2.rds.amazonaws.com
  default.myrds-port: "3306"
  default.myrds-user: admin
kind: ConfigMap
metadata:
  name: myrds-conn-cm
  namespace: default

# fieldexport 정보 확인
kubectl get crd | grep fieldexport
fieldexports.services.k8s.aws                   2023-06-04T12:46:39Z

kubectl get fieldexport
NAME         AGE
myrds-host   3m8s
myrds-port   3m8s
myrds-user   3m8s

kubectl get fieldexport myrds-host -o yaml | k neat | yh

APP_NAMESPACE=default
cat <<EOF > rds-pods.yaml
apiVersion: v1
kind: Pod
metadata:
  name: app
  namespace: ${APP_NAMESPACE}
spec:
  containers:
   - image: busybox
     name: myapp
     command:
        - sleep
        - "3600"
     imagePullPolicy: IfNotPresent
     env:
      - name: DBHOST
        valueFrom:
         configMapKeyRef:
          name: ${RDS_INSTANCE_CONN_CM}
          key: "${APP_NAMESPACE}.${RDS_INSTANCE_NAME}-host"
      - name: DBPORT
        valueFrom:
         configMapKeyRef:
          name: ${RDS_INSTANCE_CONN_CM}
          key: "${APP_NAMESPACE}.${RDS_INSTANCE_NAME}-port"
      - name: DBUSER
        valueFrom:
         configMapKeyRef:
          name: ${RDS_INSTANCE_CONN_CM}
          key: "${APP_NAMESPACE}.${RDS_INSTANCE_NAME}-user"
      - name: DBPASSWORD
        valueFrom:
          secretKeyRef:
           name: "${RDS_INSTANCE_NAME}-password"
           key: password
EOF
kubectl apply -f rds-pods.yaml

# 생성 확인
kubectl get pod app
NAME   READY   STATUS    RESTARTS   AGE
app    1/1     Running   0          5s

# 파드의 환경 변수 확인
kubectl exec -it app -- env | grep DB
DBUSER=admin
DBPASSWORD=qwe12345
DBHOST=myrds.cnlc7l....ap-northeast-2.rds.amazonaws.com
DBPORT=3306

# [터미널]
watch -d "kubectl get dbinstance; echo; kubectl get cm myrds-conn-cm -o yaml | kubectl neat"

apiVersion: v1
data:
  default.myrds-host: myrds.cnlc7....ap-northeast-2.rds.amazonaws.com
  default.myrds-port: "3306"
  default.myrds-user: admin

# DB 식별자를 업데이트 >> 어떤 현상이 발생하는가?
kubectl patch dbinstance myrds --type=merge -p '{"spec":{"dbInstanceIdentifier":"studyend"}}'
# 상태가 creating -> backing-up -> available로 변경되면서 DB식별자도 myrds에서 studyend로 업데이트 됨.
NAME    STATUS
myrds   creating
NAME    STATUS
myrds   backing-up
NAME    STATUS
myrds   available
apiVersion: v1
data:
  default.myrds-host: studyend.cnlc7...ap-northeast-2.rds.amazonaws.com

# 확인
kubectl get dbinstance myrds
NAME    STATUS
myrds   available

kubectl describe dbinstance myrds
Name:         myrds
Namespace:    default
...
  Endpoint:
    Address:                            studyend.cnlc7lcs9gjt.ap-northeast-2.rds.amazonaws.com
...

# 상태 정보 확인 : address 변경 확인!
kubectl get dbinstances myrds -o jsonpath={.status.endpoint} | jq
{
  "address": "studyend.cnlc7lc....ap-northeast-2.rds.amazonaws.com",
  "hostedZoneID": "ZLA2NU...",
  "port": 3306
}

# 파드의 환경 변수 확인 >> 파드의 경우 환경 변수 env로 정보를 주입했기 때문에 변경된 정보를 확인 할 수 없다
kubectl exec -it app -- env | grep DB
DBHOST=myrds.cnlc7lc....ap-northeast-2.rds.amazonaws.com
DBPORT=3306
DBUSER=admin
DBPASSWORD=qwe12345

# 파드 삭제 후 재생성 후 확인
kubectl delete pod app && kubectl apply -f rds-pods.yaml

# 파드의 환경 변수 확인 >> 변경 정보 확인!
# 즉 deployments, daemonsets, statefulsets 의 경우 rollout 으로 env 변경 적용을 할 수 는 있겠다!
kubectl exec -it app -- env | grep DB
DBHOST=studyend.cnlc....ap-northeast-2.rds.amazonaws.com
DBPORT=3306
DBUSER=admin
DBPASSWORD=qwe12345

# 파드 삭제
kubectl delete pod app

# RDS 삭제 
kubectl delete -f rds-mariadb.yaml

# db식별자 myrds 삭제는 AWS CLI나 AWS Console에서 진행
aws rds delete-db-instance --db-instance-identifier myrds --skip-final-snapshot

# 서비스명 변수 지정 및 helm 차트 다운로드
export SERVICE=dynamodb
export RELEASE_VERSION=$(curl -sL https://api.github.com/repos/aws-controllers-k8s/$SERVICE-controller/releases/latest | grep '"tag_name":' | cut -d'"' -f4 | cut -c 2-)
helm pull oci://public.ecr.aws/aws-controllers-k8s/$SERVICE-chart --version=$RELEASE_VERSION
tar xzvf $SERVICE-chart-$RELEASE_VERSION.tgz

# helm chart 확인
tree ~/$SERVICE-chart

# ACK dynamodb-Controller 설치
export ACK_SYSTEM_NAMESPACE=ack-system
export AWS_REGION=ap-northeast-2
helm install -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller --set aws.region="$AWS_REGION" ~/$SERVICE-chart

# 설치 확인
helm list --namespace $ACK_SYSTEM_NAMESPACE
NAME                 	NAMESPACE 	REVISION	UPDATED                                	STATUS  	CHART             	APP VERSION
ack-ec2-controller   	ack-system	1       	2023-06-04 21:46:39.80730411 +0900 KST 	deployed	ec2-chart-1.0.3   	1.0.3
ack-dynamodb-controller	ack-system	1       	2023-06-05 11:22:44.98087282 +0900 KST 	deployed	dynamodb-chart-1.1.1	1.1.1
ack-rds-controller   	ack-system	1       	2023-06-05 08:24:20.821985798 +0900 KST	deployed	rds-chart-1.1.4   	1.1.4

kubectl -n $ACK_SYSTEM_NAMESPACE get pods -l "app.kubernetes.io/instance=ack-$SERVICE-controller"
NAME                                                      READY   STATUS    RESTARTS   AGE
ack-dynamodb-controller-dynamodb-chart-779c6458d8-wq7tc   1/1     Running   0          6m31s

kubectl get crd | grep $SERVICE
backups.dynamodb.services.k8s.aws               2023-06-05T02:22:44Z
globaltables.dynamodb.services.k8s.aws          2023-06-05T02:22:44Z
tables.dynamodb.services.k8s.aws                2023-06-05T02:22:44Z

# Create an iamserviceaccount - AWS IAM role bound to a Kubernetes service account
eksctl create iamserviceaccount \
  --name ack-$SERVICE-controller \
  --namespace $ACK_SYSTEM_NAMESPACE \
  --cluster $CLUSTER_NAME \
  --attach-policy-arn $(aws iam list-policies --query 'Policies[?PolicyName==`AmazonDynamoDBFullAccess`].Arn' --output text) \
  --override-existing-serviceaccounts --approve

# 확인 >> 웹 관리 콘솔에서 CloudFormation Stack >> IAM Role 확인
eksctl get iamserviceaccount --cluster $CLUSTER_NAME
NAMESPACE	NAME				ROLE ARN
ack-system	ack-dynamodb-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-199XY74UI3PSY
ack-system	ack-ec2-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-VK23PW7Y288N
ack-system	ack-rds-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-AUTL0REK98XM
ack-system	ack-s3-controller		arn:aws:iam::MyAccount:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-XFQ0LIYFYJ3Z

# Inspecting the newly created Kubernetes Service Account, we can see the role we want it to assume in our pod.
kubectl get sa -n $ACK_SYSTEM_NAMESPACE
NAME                    SECRETS   AGE
ack-dynamodb-controller   0         116s
ack-ec2-controller      0         12h
ack-rds-controller      0         124m

kubectl describe sa ack-$SERVICE-controller -n $ACK_SYSTEM_NAMESPACE
Name:                ack-dynamodb-controller
Namespace:           ack-system
Labels:              app.kubernetes.io/managed-by=eksctl
Annotations:         eks.amazonaws.com/role-arn: arn:aws:iam::587122150371:role/eksctl-myeks-addon-iamserviceaccount-ack-sys-Role1-199XY74UI3PSY

# Restart ACK service controller deployment using the following commands.
kubectl -n $ACK_SYSTEM_NAMESPACE rollout restart deploy ack-$SERVICE-controller-$SERVICE-chart
deployment.apps/ack-dynamodb-controller-dynamodb-chart restarted

# IRSA 적용으로 Env, Volume 추가 확인
kubectl describe pod -n $ACK_SYSTEM_NAMESPACE -l k8s-app=$SERVICE-chart
Name:                      ack-dynamodb-controller-dynamodb-chart-779c6458d8-wq7tc
Namespace:                 ack-system
Priority:                  0
Service Account:           ack-dynamodb-controller
Node:                      ip-192-168-1-10.ap-northeast-2.compute.internal/192.168.1.10
...

# DynamoDB 배포
cat <<EOF > dynamodb.yaml
apiVersion: dynamodb.services.k8s.aws/v1alpha1
kind: Table
metadata:
  name: my-dynamodb-table
  namespace: ack-system
spec:
  tableName: my-dynamodb-table
  billingMode: PAY_PER_REQUEST
  attributeDefinitions:
    - attributeName: id
      attributeType: S
  keySchema:
    - attributeName: id
      keyType: HASH
EOF
kubectl apply -f dynamodb.yaml
table.dynamodb.services.k8s.aws/my-dynamodb-table created

# AWS CLI 통해 배포 확인
aws dynamodb list-tables
{
    "TableNames": [
        "my-dynamodb-table"
    ]
}

# kubectl ACK 통해 확인
kubectl get table -n ack-system
NAME                CLASS   STATUS   SYNCED   AGE
my-dynamodb-table           ACTIVE   True     2m36s

# Tag을 입력하기 위한 Yaml 작성 및 배포
cat <<EOF > dynamodb-update.yaml
apiVersion: dynamodb.services.k8s.aws/v1alpha1
kind: Table
metadata:
  name: my-dynamodb-table
  namespace: ack-system
spec:
  tableName: my-dynamodb-table
  billingMode: PAY_PER_REQUEST
  attributeDefinitions:
    - attributeName: id
      attributeType: S
  keySchema:
    - attributeName: id
      keyType: HASH
  tags:
    - key: myTagKey
      value: myNewTagValue
EOF
kubectl apply -f dynamodb-update.yaml

# dynamodb Tag 확인
ddbARN=$(aws dynamodb describe-table --table-name my-dynamodb-table --query 'Table.TableArn' --output text)
echo $ddbARN

aws dynamodb list-tags-of-resource --resource-arn $ddbARN
{
    "Tags": [
        {
            "Key": "myTagKey",
            "Value": "myNewTagValue"
        },
        {
            "Key": "services.k8s.aws/controller-version",
            "Value": "dynamodb-v1.1.1"
        },
        {
            "Key": "services.k8s.aws/namespace",
            "Value": "ack-system"
        }
    ]
}

 

# dynamodb table 삭제
kubectl delete table my-dynamodb-table -n ack-system
table.dynamodb.services.k8s.aws "my-dynamodb-table" deleted

kubectl get table -n ack-system
No resources found in ack-system namespace.

aws dynamodb list-tables
{
    "TableNames": []
}

# ACK ddb controller 및 IRSA 삭제
helm uninstall -n $ACK_SYSTEM_NAMESPACE ack-$SERVICE-controller
kubectl delete -f ~/$SERVICE-chart/crds
eksctl delete iamserviceaccount --cluster myeks --name ack-$SERVICE-controller --namespace ack-system

# Flux CLI 설치
curl -s https://fluxcd.io/install.sh | sudo bash
[INFO]  Downloading metadata https://api.github.com/repos/fluxcd/flux2/releases/latest
[INFO]  Using 2.0.0-rc.5 as release
[INFO]  Downloading hash https://github.com/fluxcd/flux2/releases/download/v2.0.0-rc.5/flux_2.0.0-rc.5_checksums.txt
[INFO]  Downloading binary https://github.com/fluxcd/flux2/releases/download/v2.0.0-rc.5/flux_2.0.0-rc.5_linux_amd64.tar.gz
[INFO]  Verifying binary download
which: no shasum in (/sbin:/bin:/usr/sbin:/usr/bin)
[INFO]  Installing flux to /usr/local/bin/flux
. <(flux completion bash)

# 버전 확인
flux --version
flux version 2.0.0-rc.5

# 자신의 Github 토큰과 유저이름 변수 지정
export GITHUB_TOKEN=<your-token>
export GITHUB_USER=<your-username>
export GITHUB_TOKEN=ghp_###
export GITHUB_USER=myname

# Bootstrap
## Creates a git repository fleet-infra on your GitHub account.
## Adds Flux component manifests to the repository.
## Deploys Flux Components to your Kubernetes Cluster.
## Configures Flux components to track the path /clusters/my-cluster/ in the repository.
flux bootstrap github \
  --owner=$GITHUB_USER \
  --repository=fleet-infra \
  --branch=main \
  --path=./clusters/my-cluster \
  --personal
✔ Kustomization reconciled successfully
► confirming components are healthy
✔ helm-controller: deployment ready
✔ kustomize-controller: deployment ready
✔ notification-controller: deployment ready
✔ source-controller: deployment ready
✔ all components are healthy

# 설치 확인
kubectl get pods -n flux-system
NAME                                       READY   STATUS    RESTARTS   AGE
helm-controller-fbdd59577-chxns            1/1     Running   0          48s
kustomize-controller-6b67b54cf8-mbb8z      1/1     Running   0          48s
notification-controller-78f4869c94-tftpm   1/1     Running   0          48s
source-controller-75db64d9f7-rdjw8         1/1     Running   0          48s

kubectl get-all -n flux-system
kubectl get crd | grep fluxc
alerts.notification.toolkit.fluxcd.io           2023-06-05T10:19:45Z
buckets.source.toolkit.fluxcd.io                2023-06-05T10:19:45Z
gitrepositories.source.toolkit.fluxcd.io        2023-06-05T10:19:45Z
helmcharts.source.toolkit.fluxcd.io             2023-06-05T10:19:45Z
helmreleases.helm.toolkit.fluxcd.io             2023-06-05T10:19:46Z
helmrepositories.source.toolkit.fluxcd.io       2023-06-05T10:19:46Z
kustomizations.kustomize.toolkit.fluxcd.io      2023-06-05T10:19:46Z
ocirepositories.source.toolkit.fluxcd.io        2023-06-05T10:19:46Z
providers.notification.toolkit.fluxcd.io        2023-06-05T10:19:46Z
receivers.notification.toolkit.fluxcd.io        2023-06-05T10:19:46Z

kubectl get gitrepository -n flux-system
NAME          URL                                       AGE   READY   STATUS
flux-system   ssh://git@github.com/myname/fleet-infra   74s   True    stored artifact for revision 'main@sha1:ede8721252d83c8f4....'

# gitops 도구 설치
curl --silent --location "https://github.com/weaveworks/weave-gitops/releases/download/v0.24.0/gitops-$(uname)-$(uname -m).tar.gz" | tar xz -C /tmp
sudo mv /tmp/gitops /usr/local/bin
gitops version
Current Version: 0.24.0
GitCommit: cc1d0e680c55e0aaf5bfa0592a0a454fb2064bc1
BuildTime: 2023-05-24T16:29:14Z
Branch: releases/v0.24.0

# flux 대시보드 설치
PASSWORD="password"
gitops create dashboard ww-gitops --password=$PASSWORD
✔ Flux &{v2.0.0-rc.5  flux-system} is already installed
► Applying GitOps Dashboard manifests
► Installing the GitOps Dashboard ...
✔ GitOps Dashboard has been installed
► Request reconciliation of dashboard (timeout 3m0s) ...
◎ Waiting for GitOps Dashboard reconciliation
✔ GitOps Dashboard ww-gitops is ready
✔ Installed GitOps Dashboard

# 확인
flux -n flux-system get helmrelease
NAME     	REVISION	SUSPENDED	READY	MESSAGE
ww-gitops	4.0.22  	False    	True 	Release reconciliation succeeded

kubectl -n flux-system get pod,svc

CERT_ARN=`aws acm list-certificates --query 'CertificateSummaryList[].CertificateArn[]' --output text`
echo $CERT_ARN

# Ingress 설정

cat <<EOT > gitops-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: gitops-ingress
  annotations:
    alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN
    alb.ingress.kubernetes.io/group.name: study
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
    alb.ingress.kubernetes.io/load-balancer-name: myeks-ingress-alb
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/ssl-redirect: "443"
    alb.ingress.kubernetes.io/success-codes: 200-399
    alb.ingress.kubernetes.io/target-type: ip
spec:
  ingressClassName: alb
  rules:
  - host: gitops.$MyDomain
    http:
      paths:
      - backend:
          service:
            name: ww-gitops-weave-gitops
            port:
              number: 9001
        path: /
        pathType: Prefix
EOT
kubectl apply -f gitops-ingress.yaml -n flux-system

# 배포 확인
kubectl get ingress -n flux-system

# GitOps 접속 정보 확인 >> 웹 접속 후 정보 확인
echo -e "GitOps Web https://gitops.$MyDomain"

# 소스 생성 : 유형 - git, helm, oci, bucket
# flux create source {소스 유형}
# 악분(최성욱)님이 준비한 repo로 git 소스 생성
GITURL="https://github.com/sungwook-practice/fluxcd-test.git"
flux create source git nginx-example1 --url=$GITURL --branch=main --interval=30s
✚ generating GitRepository source
► applying GitRepository source
✔ GitRepository source created
◎ waiting for GitRepository source reconciliation
✔ GitRepository source reconciliation completed
✔ fetched revision: main@sha1:4478b54cb...

# 소스 확인
flux get sources git
NAME          	REVISION          	SUSPENDED	READY	MESSAGE
flux-system   	main@sha1:ede87212	False    	True 	stored artifact for revision 'main@sha1:ede87212'
nginx-example1	main@sha1:4478b54c	False    	True 	stored artifact for revision 'main@sha1:4478b54c'

kubectl -n flux-system get gitrepositories
NAME             URL                                                    AGE    READY   STATUS
flux-system      ssh://git@github.com/myname/fleet-infra                137m   True    stored artifact for revision 'main@sha1:ede8721252d...'
nginx-example1   https://github.com/sungwook-practice/fluxcd-test.git   102s   True    stored artifact for revision 'main@sha1:4478b54cb7a...'

# [터미널] 모니터링
watch -d kubectl get pod,svc nginx-example1

# flux 애플리케이션 생성 : nginx-example1
flux create kustomization nginx-example1 --target-namespace=default --interval=1m --source=nginx-example1 --path="./nginx" --health-check-timeout=2m

# 확인
kubectl get pod,svc nginx-example1
kubectl get kustomizations -n flux-system
flux get kustomizations

# [터미널] 모니터링
watch -d kubectl get pod,svc nginx-example1

# flux 애플리케이션 삭제 >> 파드와 서비스는? flux 애플리케이션 생성 시 --prune 옵션 false(default 값)
flux delete kustomization nginx-example1
# Pod와 서비스는 삭제되지 않고 Application만 삭제 된다. --prune 옵션 false이기 때문에! 다만, gitops 대시보드에는 삭제 된 것으로 나타난다.
flux get kustomizations
NAME       	REVISION          	SUSPENDED	READY	MESSAGE
flux-system	main@sha1:ede87212	False    	True 	Applied revision: main@sha1:ede87212

kubectl get pod,svc nginx-example1

# flux 애플리케이션 다시 생성 :  --prune 옵션 true
flux create kustomization nginx-example1 \
  --target-namespace=default \
  --prune=true \
  --interval=1m \
  --source=nginx-example1 \
  --path="./nginx" \
  --health-check-timeout=2m

# 확인
flux get kustomizations
NAME          	REVISION          	SUSPENDED	READY	MESSAGE
flux-system   	main@sha1:ede87212	False    	True 	Applied revision: main@sha1:ede87212
nginx-example1	main@sha1:4478b54c	False    	True 	Applied revision: main@sha1:4478b54c

kubectl get pod,svc nginx-example1

# flux 애플리케이션 삭제 >> 파드와 서비스는? 
flux delete kustomization nginx-example1
# Pod와 서비스 모두 삭제 된다. --prune 옵션이 true이기 때문에!
flux get kustomizations
kubectl get pod,svc nginx-example1

# flux 소스 삭제
flux delete source git nginx-example1

# 소스 확인
flux get sources git
kubectl -n flux-system get gitrepositories

# flux 삭제

# 사용 리전의 인증서 ARN 확인
aws acm list-certificates --query 'CertificateSummaryList[].CertificateArn[]' --output text
CERT_ARN=`aws acm list-certificates --query 'CertificateSummaryList[].CertificateArn[]' --output text`
echo "alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN"

# 하버 설치<
helm repo add harbor https://helm.goharbor.io
helm fetch harbor/harbor --untar --version 1.11.0
vim ~/harbor/values.yaml
----------------------
expose.tls.certSource=none                        # 19줄
expose.ingress.hosts.core=harbor.<각자자신의도메인>    # 36줄
expose.ingress.hosts.notary=notary.<각자자신의도메인>  # 37줄<
expose.ingress.hosts.core=harbor.bs-yang.com
expose.ingress.hosts.notary=notary.bs-yang.com
expose.ingress.controller=alb                      # 44줄
expose.ingress.className=alb                       # 47줄~
expose.ingress.annotations=alb.ingress.kubernetes.io/scheme: internet-facing
expose.ingress.annotations=alb.ingress.kubernetes.io/target-type: ip
expose.ingress.annotations=alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
expose.ingress.annotations=alb.ingress.kubernetes.io/certificate-arn: ${CERT_ARN}   # 각자 자신의 값으로 수정입력
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
alb.ingress.kubernetes.io/certificate-arn: ${CERT_ARN}   # 각자 자신의 값으로 수정입력
externalURL=https://harbor.<각자자신의도메인>          # 131줄
externalURL=https://harbor.bs-yang.com             
----------------------
kubectl create ns harbor

helm install harbor harbor/harbor -f ~/harbor/values.yaml --namespace harbor --version 1.11.0

# 컨테이너 이미지 가져오기
docker pull nginx && docker pull busybox && docker images
REPOSITORY   TAG       IMAGE ID       CREATED       SIZE
nginx        latest    f9c14fe76d50   11 days ago   143MB
busybox      latest    8135583d97fe   2 weeks ago   4.86MB

# 태그 설정
docker tag busybox harbor.$MyDomain/aews/busybox:0.1
docker image ls
REPOSITORY                        TAG       IMAGE ID       CREATED       SIZE
nginx                             latest    f9c14fe76d50   11 days ago   143MB
busybox                           latest    8135583d97fe   2 weeks ago   4.86MB
harbor.bs-yang.com/aews/busybox   0.1       8135583d97fe   2 weeks ago   4.86MB

# 로그인 - 비밀번호는 미리 Harbor Portal에서 변경을 한다. 아래는 기본값을 바탕으로 한다.
echo 'Harbor12345' > harborpw.txt
cat harborpw.txt | docker login harbor.$MyDomain -u admin --password-stdin
cat /root/.docker/config.json | jq

# 이미지 업로드
docker push harbor.$MyDomain/aews/busybox:0.1
The push refers to repository [harbor.bs-yang.com/aews/busybox]
9547b4c33213: Pushed
0.1: digest: sha256:5cd3db04b8be5773388576a83177aff4f40a03457a63855f4b9cbe30542b9a43 size: 528

# 파드 배포
curl -s -O https://raw.githubusercontent.com/junghoon2/kube-books/main/ch13/busybox-deploy.yml
sed -i "s|harbor.myweb.io/erp|harbor.$MyDomain/aews|g" busybox-deploy.yml
kubectl apply -f busybox-deploy.yml
NAME                      READY   STATUS    RESTARTS   AGE
busybox-7494977b8-bpgs7   1/1     Running   0          3s

# 태그 설정
docker tag nginx harbor.$MyDomain/aews/nginx:0.1
docker image ls
REPOSITORY                        TAG       IMAGE ID       CREATED       SIZE
nginx                             latest    f9c14fe76d50   12 days ago   143MB
harbor.bs-yang.com/aews/nginx     0.1       f9c14fe76d50   12 days ago   143MB
busybox                           latest    8135583d97fe   2 weeks ago   4.86MB
harbor.bs-yang.com/aews/busybox   0.1       8135583d97fe   2 weeks ago   4.86MB

# 이미지 업로드
docker push harbor.$MyDomain/aews/nginx:0.1

# 모니터링
kubectl create ns gitlab
watch kubectl get pod,pvc,ingress -n gitlab

# 설치
echo $CERT_ARN
helm repo add gitlab https://charts.gitlab.io/
helm repo update
helm fetch gitlab/gitlab --untar --version 6.8.1
vim ~/gitlab/values.yaml
----------------------
global:
  hosts:
    domain: <각자자신의도메인>             # 52줄
    https: true

  ingress:                             # 66줄~
    configureCertmanager: false
    provider: aws
    class: alb
    annotations:
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
      alb.ingress.kubernetes.io/certificate-arn: ${CERT_ARN}   # 각자 자신의 값으로 수정입력
      alb.ingress.kubernetes.io/success-codes: 200-399
      alb.ingress.kubernetes.io/group.name: "gitlab" # 이렇게 할 경우 4개의 Ingress을 하나의 ALB로 생성 가능<
    tls:                               # 79줄
      enabled: false
----------------------
helm install gitlab gitlab/gitlab -f ~/gitlab/values.yaml --set certmanager.install=false --set nginx-ingress.enabled=false --set prometheus.install=false --set gitlab-runner.install=false --namespace gitlab --version 6.8.4

# 확인 - SubCharts
# gitlab-gitaly : 웹서비스 혹은 ssh 방식으로 진행되는 깃 제목, 브랜치, 태그 등의 깃 요청 등에 대한 작업을 담당
# gitlab-gitlab-shell : https 가 아닌 ssh 방식으로 깃 명령어 실행 시 해당 요청을 처리
# gitlab-kas : gitlab agent server
# gitlab-postgresql : 유저, 권한, 이슈 등 깃랩의 메타 데이터 정보가 저장
# gitlab-redis-master : 깃랩 작업 정보는 레디스 캐시 서버를 이용하여 처리
# gitlab-sidekiq-all-in-1-v2 : 레디스와 연동하여 작업 큐 처리 용도로 사용
# gitlab-webservice-default : 깃랩 웹 서비스를 처리
helm list -n gitlab
NAME  	NAMESPACE	REVISION	UPDATED                                	STATUS  	CHART       	APP VERSION
gitlab	gitlab   	1       	2023-06-06 21:08:33.473720611 +0900 KST	deployed	gitlab-6.8.4	15.8.4

kubectl get pod,pvc,ingress,deploy,sts -n gitlab
kubectl df-pv -n gitlab
kubectl get-all -n gitlab

# 웹 root 계정 암호 확인
kubectl get secrets -n gitlab gitlab-gitlab-initial-root-password --template={{.data.password}} | base64 -d ;echo

#
mkdir ~/gitlab-test && cd ~/gitlab-test

# git 계정 초기화 : 토큰 및 로그인 실패 시 매번 실행해주자
git config --system --unset credential.helper
git config --global --unset credential.helper

# git 계정 정보 확인 및 global 계정 정보 입력
git config --list
git config --global user.name "<각자 자신의 Gialba 계정>"
git config --global user.email "<각자 자신의 Gialba 계정의 이메일>"
git config --global user.name "myname"
git config --global user.email "mymail@mail.net"

# git clone
git clone https://gitlab.$MyDomain/<각자 자신의 Gitlab 계정>/test-stg.git
git clone https://gitlab.$MyDomain/myname/test-stg.git
Cloning into 'test-stg'...
Username for 'https://gitlab.bs-yang.com': bsyang
Password for 'https://bsyang@gitlab.bs-yang.com': (토근입력)
remote: Enumerating objects: 3, done.
remote: Counting objects: 100% (3/3), done.
remote: Compressing objects: 100% (2/2), done.
remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
Receiving objects: 100% (3/3), done.

# 이동
ls -al test-stg && cd test-stg && pwd
total 8
drwxr-xr-x 3 root root   35 Jun  7 13:20 .
drwxr-xr-x 3 root root   22 Jun  7 13:18 ..
drwxr-xr-x 8 root root  163 Jun  7 13:20 .git
-rw-r--r-- 1 root root 6207 Jun  7 13:20 README.md
/root/gitlab-test/test-stg

# 파일 생성 및 깃 업로드(push) : 웹에서 확인
echo "gitlab test memo" >> test.txt
git add . && git commit -m "initial commit - add test.txt"
git push
Username for 'https://gitlab.bs-yang.com': bsyang
Password for 'https://bsyang@gitlab.bs-yang.com': (토근입력)
Enumerating objects: 4, done.
Counting objects: 100% (4/4), done.
Delta compression using up to 2 threads
Compressing objects: 100% (2/2), done.
Writing objects: 100% (3/3), 299 bytes | 299.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
To https://gitlab.bs-yang.com/bsyang/test-stg.git
   33958db..5075bf6  main -> main

# 모니터링 [터미널2]
kubectl create ns argocd
watch kubectl get pod,pvc,svc -n argocd

# 설치
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

# 설치 확인
# argocd-application-controller : 실행 중인 k8s 애플리케이션의 설정과 깃 저장소의 소스 파일에 선언된 상태를 서로 비교하는 컨트롤러. 상태와 다르면 ‘OutOfSync’ 에러를 출력.
# argocd-dex-server : 외부 사용자의 LDAP 인증에 Dex 서버를 사용할 수 있음
# argocd-repo-server : 원격 깃 저장소의 소스 코드를 아르고시디 내부 캐시 서버에 저장합니다. 디렉토리 경로, 소스, 헬름 차트 등이 저장.
kubectl get pod,pvc,svc,deploy,sts -n argocd
kubectl get-all -n argocd

kubectl get crd | grep argoproj

# 서비스 노출을 위해 서비스 타임 NodePort로 변경
kubectl patch svc argocd-server -n argocd -p '{"spec": {"type": "NodePort"}}'

# Ingress 설정 및 설치
CERT_ARN=`aws acm list-certificates --query 'CertificateSummaryList[].CertificateArn[]' --output text`
echo $CERT_ARN

cat <<EOF > argocd-ingress-set.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: argocd-ingress
  namespace: argocd
  annotations:
    kubernetes.io/ingress.class: alb
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
    alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN
    alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
    alb.ingress.kubernetes.io/backend-protocol: HTTPS
    alb.ingress.kubernetes.io/healthcheck-path: /login
spec:
  rules:
    - host: argocd.$MyDomain
      http:
        paths:
          - pathType: Prefix
            path: /
            backend:
              service:
                name: argocd-server
                port:
                  number: 443
EOF
kubectl apply -f argocd-ingress-set.yaml -n argocd

# ingress 설정 확인
kubectl get pod,pvc,svc,deploy,sts,ingress -n argocd

# admin 계정의 암호 확인
ARGOPW=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d)
echo $ARGOPW

# 웹 접속 로그인 (admin) CLB의 DNS 주소로 접속
echo -e "Argocd Web URL = https://argocd.$MyDomain"

# 최신버전 설치
curl -sSL -o argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
install -m 555 argocd-linux-amd64 /usr/local/bin/argocd
chmod +x /usr/local/bin/argocd

# 버전 확인
argocd version --short
argocd: v2.7.4+a33baa3
FATA[0000] Argo CD server address unspecified

# Help
# argocd app : 쿠버네티스 애플리케이션 동기화 상태 확인
# argocd context : 복수의 쿠버네티스 클러스터 등록 및 선택
# argocd login : 아르고시디 서버에 로그인 
# argocd repo : 원격 깃 저장소를 등록하고 현황 파악
argocd

# argocd 서버 로그인
argocd login argocd.$MyDomain --username admin --password $ARGOPW
'admin:login' logged in successfully
Context 'argocd.bs-yang.com' updated

# 기 설치한 깃랩의 프로젝트 URL 을 argocd 깃 리포지토리(argocd repo)로 등록. 깃랩은 프로젝트 단위로 소스 코드를 보관.
argocd repo add https://gitlab.$MyDomain/myname/test-stg.git --username myname --password <깃랩 계정 암호>
Repository 'https://gitlab.bs-yang.com/myname/test-stg.git' added

# gitlab 등록 확인
argocd repo list
TYPE  NAME  REPO                                            INSECURE  OCI    LFS    CREDS  STATUS      MESSAGE  PROJECT
git         https://gitlab.bs-yang.com/myname/test-stg.git  false     false  false  true   Successful

# test-stg 깃 디렉터리에서 아래 실행
cd ~/gitlab-test/test-stg

# 깃 원격 오리진 주소 확인
git config -l | grep remote.origin.url
remote.origin.url=https://gitlab.bs-yang.com/myname/test-stg.git

# RabbitMQ 헬름 차트 설치
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm fetch bitnami/rabbitmq --untar --version 11.10.3
cd rabbitmq/
cp values.yaml my-values.yaml

# 헬름 차트를 깃랩 저장소에 업로드
git add . && git commit -m "add rabbitmq helm"
git push
Username for 'https://gitlab.bs-yang.com': myname
Password for 'https://bsyang@gitlab.bs-yang.com':
Enumerating objects: 57, done.
Counting objects: 100% (57/57), done.
Delta compression using up to 16 threads
Compressing objects: 100% (54/54), done.
Writing objects: 100% (56/56), 65.14 KiB | 6.51 MiB/s, done.
Total 56 (delta 13), reused 0 (delta 0), pack-reused 0
To https://gitlab.bs-yang.com/bsyang/test-stg.git
   856ff34..c96d6cb  main -> main

# 수정
cd ~/
curl -s -O https://raw.githubusercontent.com/wikibook/kubepractice/main/ch15/rabbitmq-helm-argo-application.yml
vim rabbitmq-helm-argo-application.yml
--------------------------------------
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: rabbitmq-helm
  namespace: argocd
  finalizers:
  - resources-finalizer.argocd.argoproj.io
spec:
  destination:
    namespace: rabbitmq
    server: https://kubernetes.default.svc
  project: default
  source:
    repoURL: https://gitlab.myurl.com/path/xxx.git #내 주소로 변경
    path: rabbitmq
    targetRevision: HEAD
    helm:
      valueFiles:
      - my-values.yaml
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
--------------------------------------

# 모니터링 : argocd 웹 화면 보고 있기!
echo -e "Argocd Web URL = https://argocd.$MyDomain"

# 배포
kubectl apply -f rabbitmq-helm-argo-application.yml

# yaml 파일 배포 후 상태 확인 (OutOfSync 상태)
kubectl get applications.argoproj.io -n argocd
NAME            SYNC STATUS   HEALTH STATUS
rabbitmq-helm   OutOfSync     Missing

# sync 후 상태 확인
NAME            SYNC STATUS   HEALTH STATUS
rabbitmq-helm   Synced        Healthy

# sts 파드 1개에서 2개로 증가 설정 후 argocd 웹 화면 모니터링
kubectl scale statefulset -n rabbitmq rabbitmq-helm --replicas 2

# helm chart 통한 설치
helm repo add crossplane-stable https://charts.crossplane.io/stable
helm repo update
kubectl create namespace  crossplane-system
helm install crossplane --namespace crossplane-system crossplane-stable/crossplane

# 설치 확인
kubectl get all -n crossplane-system
NAME                                           READY   STATUS    RESTARTS   AGE
pod/crossplane-9f6d5cd7b-5x9np                 1/1     Running   0          64s
pod/crossplane-rbac-manager-699dc89cf4-vck8n   1/1     Running   0          64s

NAME                          TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
service/crossplane-webhooks   ClusterIP   10.100.175.230   <none>        9443/TCP   64s

NAME                                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/crossplane                1/1     1            1           64s
deployment.apps/crossplane-rbac-manager   1/1     1            1           64s

NAME                                                 DESIRED   CURRENT   READY   AGE
replicaset.apps/crossplane-9f6d5cd7b                 1         1         1       64s
replicaset.apps/crossplane-rbac-manager-699dc89cf4   1         1         1       64s

# crossplane CLI 설치
curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh
sudo mv kubectl-crossplane /usr/local/bin
kubectl crossplane --help

# AWS Provider 설치
cat <<EOF | kubectl apply -f -
apiVersion: pkg.crossplane.io/v1
kind: Provider
metadata:
  name: upbound-provider-aws
spec:
  package: xpkg.upbound.io/upbound/provider-aws:v0.27.0
EOF

# provider 확인. HEALTHY가 true가 되는데까지 최대 약 5분 정도 소요
kubectl get providers
NAME                   INSTALLED   HEALTHY   PACKAGE                                        AGE
upbound-provider-aws   True        True      xpkg.upbound.io/upbound/provider-aws:v0.27.0   98s

# AWS Configure 정보를 토대로 aws-credentials.txt 파일 생성
[default]
aws_access_key_id = AKIAYRMZ...
aws_secret_access_key = DcWBPj3t...

# secret 만들기
kubectl create secret generic aws-secret -n crossplane-system --from-file=creds=./aws-credentials.txt
secret/aws-secret created

# Secret 정보 확인
kubectl describe secret -n crossplane-system
Name:         aws-secret
Namespace:    crossplane-system
...
Data
====
creds:  116 bytes
...

# Provider Config 파일 생성 수 업데이트 진행
cat <<EOF | kubectl apply -f -
apiVersion: aws.upbound.io/v1beta1
kind: ProviderConfig
metadata:
  name: default
spec:
  credentials:
    source: Secret
    secretRef:
      namespace: crossplane-system
      name: aws-secret
      key: creds
EOF
providerconfig.aws.upbound.io/default created

# Provider Config 확인
kubectl describe providerconfig.aws.upbound.io/default -n crossplane-system
Name:         default
...
Spec:
  Credentials:
    Secret Ref:
      Key:        creds
      Name:       aws-secret
      Namespace:  crossplane-system
    Source:       Secret
...

# S3 목록 확인
aws s3 ls
2023-06-07 15:26:53 cf-templates-2awcm82lq9tn-ap-northeast-2
2023-04-27 13:20:41 cloudtrail-awslogs-....-not-delete
2023-04-29 14:28:11 do-not-delete-....

kubectl get buckets
No resources found

# S3 Bucket 생성
bucket=$(echo "crossplane-bucket-"$(head -n 4096 /dev/urandom | openssl sha1 | tail -c 10))
cat <<EOF | kubectl apply -f -
apiVersion: s3.aws.upbound.io/v1beta1
kind: Bucket
metadata:
  name: $bucket
spec:
  forProvider:
    region: $AWS_REGION
  providerConfigRef:
    name: default
EOF
bucket.s3.aws.upbound.io/crossplane-bucket-9582936ef created

# S3 Bucket 생성 확인
kubectl get buckets
NAME                          READY   SYNCED   EXTERNAL-NAME                 AGE
crossplane-bucket-9582936ef   True    True     crossplane-bucket-9582936ef   14s

aws s3 ls
2023-06-07 15:26:53 cf-templates-2awc...
2023-04-27 13:20:41 cloudtrail-awslogs-...
2023-06-08 22:39:56 crossplane-bucket-9582936ef
2023-04-29 14:28:11 do-not-delete-...

# 앞서 crossplane에서 만든 bucket과 AWS Console에서 만든 Bucket(test 붙은 bucket) 목록 확인
aws s3 ls
...
2023-06-08 22:48:45 crossplane-bucket-64d76c5fa-test
2023-06-08 22:39:56 crossplane-bucket-9582936ef

# crossplane bucket 목록 확인. 앞서 Crossplane 통해 만든 Bucket만 보임
kubectl get bucket
NAME                          READY   SYNCED   EXTERNAL-NAME                 AGE
crossplane-bucket-9582936ef   True    True     crossplane-bucket-9582936ef   10m

# bucket import 진행
cat <<EOF | kubectl apply -f -
apiVersion: s3.aws.upbound.io/v1beta1
kind: Bucket
metadata:
  name: bucket-import
  annotations:
    crossplane.io/external-name: crossplane-bucket-64d76c5fa-test
spec:
  forProvider:
    region: $AWS_REGION
  providerConfigRef:
    name: default
EOF
bucket.s3.aws.upbound.io/bucket-import created

# bucket import 확인
kubectl get bucket
NAME                          READY   SYNCED   EXTERNAL-NAME                      AGE
bucket-import                 True    True     crossplane-bucket-64d76c5fa-test   76s
crossplane-bucket-9582936ef   True    True     crossplane-bucket-9582936ef        18m

# 삭제 전 Bucket 목록 확인
aws s3 ls
2023-06-08 22:57:22 crossplane-bucket-64d76c5fa-test
2023-06-08 22:39:56 crossplane-bucket-9582936ef

kubectl get bucket
NAME                          READY   SYNCED   EXTERNAL-NAME                      AGE
bucket-import                 True    True     crossplane-bucket-64d76c5fa-test   5m16s
crossplane-bucket-9582936ef   True    True     crossplane-bucket-9582936ef        22m

# crossplane으로 만든 Bucket 삭제
kubectl delete bucket crossplane-bucket-9582936ef
bucket.s3.aws.upbound.io "crossplane-bucket-9582936ef" deleted

kubectl get bucket
NAME            READY   SYNCED   EXTERNAL-NAME                      AGE
bucket-import   True    True     crossplane-bucket-64d76c5fa-test   6m4s

aws s3 ls
2023-06-08 22:57:22 crossplane-bucket-64d76c5fa-test

# AWS Console에서 만들고 crossplane으로 Import한 Bucket 삭제
kubectl delete bucket bucket-import
bucket.s3.aws.upbound.io "bucket-import" deleted

kubectl get bucket
No resources found

aws s3 ls

# 압축파일 다운로드
curl -sSL -o eksdemo_Linux_x86_64.tar.gz https://github.com/awslabs/eksdemo/releases/download/v0.8.0/eksdemo_Linux_x86_64.tar.gz

# 압축해제 및 파일 이동
tar xzvf eksdemo_Linux_x86_64.tar.gz
mv eksdemo /usr/local/bin/

ls /usr/local/bin
argocd  aws  aws_completer  eksctl  eksdemo  helm  kubectl  yh

# 설치 확인
eksdemo version
eksdemo version info: cmd.Version{Version:"0.8.0", Date:"2023-06-03T17:45:05Z", Commit:"bac7ddb"}

# cluster 정보 조회
eksdemo get cluster
+-------+--------+---------+---------+----------+----------+
|  Age  | Status | Cluster | Version | Platform | Endpoint |
+-------+--------+---------+---------+----------+----------+
| 1 day | ACTIVE | *myeks  |    1.24 | eks.7    | Public   |
+-------+--------+---------+---------+----------+----------+
* Indicates current context in local kubeconfig

# node 정보 조회
eksdemo get node -c $CLUSTER_NAME
+-------+--------------------+---------------------+------------+-----------------+-----------+
|  Age  |        Name        |     Instance Id     |    Type    |      Zone       | Nodegroup |
+-------+--------------------+---------------------+------------+-----------------+-----------+
| 1 day | ip-192-168-1-43.*  | i-0d9e7f20ce15ed4e6 | c5.4xlarge | ap-northeast-2a | ng1       |
| 1 day | ip-192-168-2-192.* | i-023ba2a4ccacca991 | c5.4xlarge | ap-northeast-2b | ng1       |
| 1 day | ip-192-168-3-73.*  | i-0a85a3af6d6bf863d | c5.4xlarge | ap-northeast-2c | ng1       |
+-------+--------------------+---------------------+------------+-----------------+-----------+
* Names end with "ap-northeast-2.compute.internal"

# dry run 통해 manifest 정보 확인
eksdemo install example-game-2048 -c $CLUSTER_NAME -I game2048.$MyDomain --dry-run
Manifest Installer Dry Run:
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: game-2048
  name: deployment-2048
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: app-2048
  replicas: 1
  template:
    metadata:
      labels:
        app.kubernetes.io/name: app-2048
    spec:
      containers:
      - image: public.ecr.aws/l6m2t8p7/docker-2048:latest
        imagePullPolicy: Always
        name: app-2048
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  namespace: game-2048
  name: service-2048
  annotations:
    {}
spec:
  ports:
    - port: 80
      targetPort: 80
      protocol: TCP
  type: ClusterIP
  selector:
    app.kubernetes.io/name: app-2048
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  namespace: game-2048
  name: ingress-2048
  annotations:
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/ssl-redirect: '443'
    alb.ingress.kubernetes.io/target-type: ip
spec:
  ingressClassName: alb
  rules:
    - host: game2048.bs-yang.com
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            service:
              name: service-2048
              port:
                number: 80
  tls:
  - hosts:
    - game2048.bs-yang.com

# game2048 설치
eksdemo install example-game-2048 -c $CLUSTER_NAME -I game2048.$MyDomain
Helm installing...
2023/06/08 16:31:40 creating 1 resource(s)
2023/06/08 16:31:40 creating 3 resource(s)
Using chart version "n/a", installed "example-game-2048" version "latest" in namespace "game-2048"

# Application 확인
eksdemo get application -c $CLUSTER_NAME
+------------------------------+-------------+---------+----------+--------+
|             Name             |  Namespace  | Version |  Status  | Chart  |
+------------------------------+-------------+---------+----------+--------+
| aws-load-balancer-controller | kube-system | v2.5.2  | deployed | 1.5.3  |
| example-game-2048            | game-2048   | latest  | deployed | n/a    |
| gitlab                       | gitlab      | 15.8.4  | deployed | 6.8.4  |
| harbor                       | harbor      | 2.7.0   | deployed | 1.11.0 |
+------------------------------+-------------+---------+----------+--------+

# ALB 확인 Provisioning -> active가 될 때까지 대기
eksdemo get load-balancer -c $CLUSTER_NAME
+------------+--------------+----------------------------------+------+-------+-----+-----+
|    Age     |    State     |               Name               | Type | Stack | AZs | SGs |
+------------+--------------+----------------------------------+------+-------+-----+-----+
| 43 seconds | provisioning | k8s-game2048-ingress2-70d50ce3fd | ALB  | ipv4  |   3 |   2 |
| 23 hours   | active       | k8s-harbor-harborin-2352dee8a2   | ALB  | ipv4  |   3 |   2 |
| 23 hours   | active       | k8s-gitlab-536957cc0a            | ALB  | ipv4  |   3 |   2 |
| 7 hours    | active       | k8s-argocd-argocdin-cc87c24740   | ALB  | ipv4  |   3 |   2 |
| 23 hours   | active       | k8s-harbor-harborin-b768b16202   | ALB  | ipv4  |   3 |   2 |
+------------+--------------+----------------------------------+------+-------+-----+-----+
* Indicates internal load balancer

# kubectl 통해 확인
kubectl get pod,pvc,svc,deploy,sts,ingress -n game-2048
NAME                                   READY   STATUS    RESTARTS   AGE
pod/deployment-2048-6bc9fd6bf5-nqg7k   1/1     Running   0          11m

NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
service/service-2048   ClusterIP   10.100.55.172   <none>        80/TCP    11m

NAME                              READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deployment-2048   1/1     1            1           11m

NAME                                     CLASS   HOSTS                  ADDRESS                                                                       PORTS     AGE
ingress.networking.k8s.io/ingress-2048   alb     game2048.bs-yang.com   k8s-game2048-ingress2-70d50ce3fd-740064416.ap-northeast-2.elb.amazonaws.com   80, 443   11m


# 사이트 주소 출력
echo "Game2048 URL : https://game2048.$MyDomain"
Game2048 URL : https://game2048.bs-yang.com