diff --git a/README.md b/README.md index 66a4149..48b67fd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ -# IaaSWeek - Ansible +# IaaSWeek - Ansible - K8S on Proxmox Ansible is a radically simple IT automation platform that makes your applications and systems easier to deploy. Avoid writing scripts or custom code to deploy and update your applications — automate in a language that approaches plain English, using SSH, with no agents to install on remote systems. ([Fonte](https://www.ansible.com/)) - [Documentação](https://docs.ansible.com/ansible/latest/index.html) - [Github](https://github.com/ansible/ansible) + +- Clona as VMs no Proxmox +- Configura o CloudInit +- \ No newline at end of file diff --git a/install_k8s/extras.yml b/install_k8s/extras.yml new file mode 100644 index 0000000..204785c --- /dev/null +++ b/install_k8s/extras.yml @@ -0,0 +1,5 @@ +- hosts: k8s_master + become: yes + user: mmagalha + roles: + - { role: install-extra, tags: ["intall_extra"]} \ No newline at end of file diff --git a/install_k8s/hosts b/install_k8s/hosts index f775a1a..1ff6d81 100644 --- a/install_k8s/hosts +++ b/install_k8s/hosts @@ -1,8 +1,19 @@ -[k8s-master] +[pve] +pve01 +[k8s] +172.32.0.221 +172.32.0.22[2:4] -[k8s-workers] +[k8s_master] +172.32.0.221 -[k8s-workers:vars] -K8S_MASTER_NODE_IP= -K8S_API_SECURE_PORT=6443 +[k8s_workers] +172.32.0.22[2:4] + +[k8s_workers:vars] +K8S_MASTER_NODE_IP=172.32.0.221 +K8S_API_SECURE_PORT=6443 + +[all:vars] +ansible_python_interpreter=/usr/bin/python3 diff --git a/install_k8s/main.yml b/install_k8s/main.yml index 90c40b8..62bd270 100644 --- a/install_k8s/main.yml +++ b/install_k8s/main.yml @@ -1,23 +1,34 @@ -- hosts: all +- hosts: pve + user: root + gather_facts: no + pre_tasks: + - name: 'atualizando repo' + raw: 'apt-get update' + - name: 'instalando o python' + raw: 'apt-get install -y python3 python3-proxmoxer' + roles: + - { role: install-vms, tags: ["install_pve_vms"]} + +- hosts: k8s become: yes - user: ubuntu + user: mmagalha gather_facts: no pre_tasks: - name: 'atualizando repo' raw: 'apt-get update' - name: 'instalando o python' - raw: 'apt-get install -y python' + raw: 'apt-get install -y python3' roles: - { role: install-k8s, tags: ["install_k8s_role"]} -- hosts: k8s-master +- hosts: k8s_master become: yes - user: ubuntu + user: mmagalha roles: - { role: create-cluster, tags: ["create_cluster_role"]} -- hosts: k8s-workers +- hosts: k8s_workers become: yes - user: ubuntu + user: mmagalha roles: - - { role: join-workers, tags: ["join_workers_role"]} + - { role: join-workers, tags: ["join_workers_role"]} diff --git a/install_k8s/roles/create-cluster/files/docker.json b/install_k8s/roles/create-cluster/files/docker.json new file mode 100644 index 0000000..92750f6 --- /dev/null +++ b/install_k8s/roles/create-cluster/files/docker.json @@ -0,0 +1,8 @@ +{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" + } \ No newline at end of file diff --git a/install_k8s/roles/create-cluster/tasks/init-cluster.yml b/install_k8s/roles/create-cluster/tasks/init-cluster.yml index 44b274f..bb4ef11 100644 --- a/install_k8s/roles/create-cluster/tasks/init-cluster.yml +++ b/install_k8s/roles/create-cluster/tasks/init-cluster.yml @@ -2,36 +2,41 @@ command: kubeadm reset --force register: kubeadmin_init + - name: Initialize Kubernetes master with kubeadm init. command: kubeadm init register: kubeadmin_init + - name: Ensure .kube directory exists. file: path: ~/.kube state: directory + - name: Symlink the kubectl admin.conf to ~/.kube/conf. file: src: /etc/kubernetes/admin.conf dest: ~/.kube/config state: link -- name: Configure weavenet networking. - shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }} - register: weavenet_result + - name: "Cluster token" shell: kubeadm token list | cut -d ' ' -f1 | sed -n '2p' register: K8S_TOKEN + - name: "CA Hash" shell: openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' register: K8S_MASTER_CA_HASH + - name: "Add K8S Token and Hash to dummy host" add_host: name: "K8S_TOKEN_HOLDER" token: "{{ K8S_TOKEN.stdout }}" hash: "{{ K8S_MASTER_CA_HASH.stdout }}" + - name: debug: msg: "[Master] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}" + - name: debug: msg: "[Master] K8S_TOKEN_HOLDER K8S Hash is {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}" diff --git a/install_k8s/roles/create-cluster/vars/main.yml b/install_k8s/roles/create-cluster/vars/main.yml index dd326c7..db7ad8d 100644 --- a/install_k8s/roles/create-cluster/vars/main.yml +++ b/install_k8s/roles/create-cluster/vars/main.yml @@ -1,3 +1,2 @@ --- -# vars file for create-cluster -default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" +# vars file for create-cluster \ No newline at end of file diff --git a/install_k8s/roles/install-extra/.travis.yml b/install_k8s/roles/install-extra/.travis.yml new file mode 100644 index 0000000..36bbf62 --- /dev/null +++ b/install_k8s/roles/install-extra/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/install_k8s/roles/install-extra/README.md b/install_k8s/roles/install-extra/README.md new file mode 100644 index 0000000..225dd44 --- /dev/null +++ b/install_k8s/roles/install-extra/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/install_k8s/roles/install-extra/defaults/main.yml b/install_k8s/roles/install-extra/defaults/main.yml new file mode 100644 index 0000000..0d146b2 --- /dev/null +++ b/install_k8s/roles/install-extra/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for install-extra \ No newline at end of file diff --git a/install_k8s/roles/install-extra/files/cert_manager_test.yaml b/install_k8s/roles/install-extra/files/cert_manager_test.yaml new file mode 100644 index 0000000..c6fadf3 --- /dev/null +++ b/install_k8s/roles/install-extra/files/cert_manager_test.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager-test +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: test-selfsigned + namespace: cert-manager-test +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: selfsigned-cert + namespace: cert-manager-test +spec: + commonName: example.com + secretName: selfsigned-cert-tls + issuerRef: + name: test-selfsigned \ No newline at end of file diff --git a/install_k8s/roles/install-extra/files/metallb_config.yaml b/install_k8s/roles/install-extra/files/metallb_config.yaml new file mode 100644 index 0000000..63caf55 --- /dev/null +++ b/install_k8s/roles/install-extra/files/metallb_config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - 172.32.0.230-172.32.0.235 \ No newline at end of file diff --git a/install_k8s/roles/install-extra/handlers/main.yml b/install_k8s/roles/install-extra/handlers/main.yml new file mode 100644 index 0000000..91caaa5 --- /dev/null +++ b/install_k8s/roles/install-extra/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for install-extra \ No newline at end of file diff --git a/install_k8s/roles/install-extra/meta/main.yml b/install_k8s/roles/install-extra/meta/main.yml new file mode 100644 index 0000000..227ad9c --- /dev/null +++ b/install_k8s/roles/install-extra/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.9 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. + \ No newline at end of file diff --git a/install_k8s/roles/install-extra/tasks/cni.yml b/install_k8s/roles/install-extra/tasks/cni.yml new file mode 100644 index 0000000..085b954 --- /dev/null +++ b/install_k8s/roles/install-extra/tasks/cni.yml @@ -0,0 +1,3 @@ +- name: Configure weavenet networking. + shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }} + register: weavenet_result \ No newline at end of file diff --git a/install_k8s/roles/install-extra/tasks/main.yml b/install_k8s/roles/install-extra/tasks/main.yml new file mode 100644 index 0000000..9c41d36 --- /dev/null +++ b/install_k8s/roles/install-extra/tasks/main.yml @@ -0,0 +1,4 @@ +--- +# tasks file for install-extra +- include: cni.yml +- include: metallb.yml \ No newline at end of file diff --git a/install_k8s/roles/install-extra/tasks/metallb.yml b/install_k8s/roles/install-extra/tasks/metallb.yml new file mode 100644 index 0000000..ebefe41 --- /dev/null +++ b/install_k8s/roles/install-extra/tasks/metallb.yml @@ -0,0 +1,18 @@ +- name: Copy configure MetalLB + copy: + src: roles/install-extra/files/metallb_config.yaml + dest: /opt/metallb_config.yaml + owner: root + group: root + backup: yes + register: metallb_install + +- name: Install MetalLB. + shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml + register: metallb_install + +- name: Install MetalLB config. + shell: kubectl apply -n metallb-system-f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml + register: metallb_install + + diff --git a/install_k8s/roles/install-extra/tests/inventory b/install_k8s/roles/install-extra/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/install_k8s/roles/install-extra/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/install_k8s/roles/install-extra/tests/test.yml b/install_k8s/roles/install-extra/tests/test.yml new file mode 100644 index 0000000..f978fae --- /dev/null +++ b/install_k8s/roles/install-extra/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - install-extra \ No newline at end of file diff --git a/install_k8s/roles/install-extra/vars/main.yml b/install_k8s/roles/install-extra/vars/main.yml new file mode 100644 index 0000000..d71758e --- /dev/null +++ b/install_k8s/roles/install-extra/vars/main.yml @@ -0,0 +1,3 @@ +--- +# vars file for install-extra +default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml index 9df4e24..afa20c0 100644 --- a/install_k8s/roles/install-k8s/tasks/install.yml +++ b/install_k8s/roles/install-k8s/tasks/install.yml @@ -1,5 +1,12 @@ -- name: Instalando o Docker - shell: curl -fsSL https://get.docker.com | bash - +- name: Adicionando as chaves do repo do Docker + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + +- name: Adicionando repositório do Docker + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" + state: present - name: Adicionando as chaves repo k8s no apt apt_key: @@ -11,11 +18,42 @@ repo: deb http://apt.kubernetes.io/ kubernetes-xenial main state: present -- name: Install k8s packages +- name: Install docker and k8s packages apt: name: "{{ packages }}" vars: packages: + - docker-ce + - docker-ce-cli + - containerd.io - kubelet - kubeadm - kubectl + +- name: Copy docker.json + copy: + src: /data/DevOps/ansible/install_k8s/roles/create-cluster/files/docker.json + dest: /etc/docker/daemon.json + owner: root + group: root + backup: yes + register: kubeadmin_init + +- name: restart systemd + systemd: + daemon_reload: yes + +- name: reload service docker, in all cases + systemd: + name: docker + state: reloaded + +- name: Disable SWAP since kubernetes can't work with swap enabled (1/2) + shell: | + swapoff -a + +- name: Disable SWAP in fstab since kubernetes can't work with swap enabled (2/2) + replace: + path: /etc/fstab + regexp: '^([^#].*?\sswap\s+sw\s+.*)$' + replace: '# \1' diff --git a/install_k8s/roles/install-vms/.travis.yml b/install_k8s/roles/install-vms/.travis.yml new file mode 100644 index 0000000..36bbf62 --- /dev/null +++ b/install_k8s/roles/install-vms/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/install_k8s/roles/install-vms/README.md b/install_k8s/roles/install-vms/README.md new file mode 100644 index 0000000..225dd44 --- /dev/null +++ b/install_k8s/roles/install-vms/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/install_k8s/roles/install-vms/defaults/main.yml b/install_k8s/roles/install-vms/defaults/main.yml new file mode 100644 index 0000000..2a65dcc --- /dev/null +++ b/install_k8s/roles/install-vms/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for install-vms \ No newline at end of file diff --git a/install_k8s/roles/install-vms/handlers/main.yml b/install_k8s/roles/install-vms/handlers/main.yml new file mode 100644 index 0000000..308c1db --- /dev/null +++ b/install_k8s/roles/install-vms/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for install-vms \ No newline at end of file diff --git a/install_k8s/roles/install-vms/meta/main.yml b/install_k8s/roles/install-vms/meta/main.yml new file mode 100644 index 0000000..227ad9c --- /dev/null +++ b/install_k8s/roles/install-vms/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.9 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. + \ No newline at end of file diff --git a/install_k8s/roles/install-vms/tasks/main.yml b/install_k8s/roles/install-vms/tasks/main.yml new file mode 100644 index 0000000..8be47de --- /dev/null +++ b/install_k8s/roles/install-vms/tasks/main.yml @@ -0,0 +1,36 @@ +--- +# tasks file for install-vm +- name: Clone VMs + proxmox_kvm: + api_user : "{{ pve_user }}" + api_password: "{{ pve_pass }}" + api_host : "{{ pve_host }}" + clone : ubnttpl # The VM source + name : k8s0{{ item }} # The target VM name + newid : 100{{ item }} + node : pve01 + storage : Local-01 + format : qcow2 + timeout : 500 +# state : started + loop: "{{ vm_id }}" + +- name: Set VM IP + command: + qm set 100{{ item }} --ipconfig0 ip=172.32.0.22{{ item }}/24,gw=172.32.0.254 + loop: "{{ vm_id }}" + ignore_errors: False + +- name: start VMs + proxmox_kvm: + api_user : "{{ pve_user }}" + api_password: "{{ pve_pass }}" + api_host : "{{ pve_host }}" + vmid : 100{{ item }} + state : started + loop: "{{ vm_id }}" + +- name: Wait 600 seconds for target connection to become reachable/usable + wait_for_connection: + delay: 600 + diff --git a/install_k8s/roles/install-vms/tests/inventory b/install_k8s/roles/install-vms/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/install_k8s/roles/install-vms/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/install_k8s/roles/install-vms/tests/test.yml b/install_k8s/roles/install-vms/tests/test.yml new file mode 100644 index 0000000..33cb5d6 --- /dev/null +++ b/install_k8s/roles/install-vms/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - install-vms \ No newline at end of file diff --git a/install_k8s/roles/install-vms/vars/main.yml b/install_k8s/roles/install-vms/vars/main.yml new file mode 100644 index 0000000..184cc2f --- /dev/null +++ b/install_k8s/roles/install-vms/vars/main.yml @@ -0,0 +1,12 @@ +--- +# vars file for install-vms +pve_host: pve01 +pve_user: root@pam +pve_pass: "{{ lookup('env','PVE_PASS') }}" +vm_id: +- 1 +- 2 +- 3 +- 4 +vm_name: "k8s0" + diff --git a/install_k8s/roles/join-workers/tasks/join-cluster.yml b/install_k8s/roles/join-workers/tasks/join-cluster.yml index 8fc172a..e9fddb4 100644 --- a/install_k8s/roles/join-workers/tasks/join-cluster.yml +++ b/install_k8s/roles/join-workers/tasks/join-cluster.yml @@ -9,7 +9,6 @@ - name: "Kubeadm reset node cluster config" command: kubeadm reset --force - register: kubeadm_reset_node - name: "Kubeadm join" shell: diff --git a/install_k8s/valt.yml b/install_k8s/valt.yml new file mode 100644 index 0000000..48d2c21 --- /dev/null +++ b/install_k8s/valt.yml @@ -0,0 +1,6 @@ +$ANSIBLE_VAULT;1.2;AES256;api_password +33343366393464373066303033613732306631393861333865353430343139363735366363396138 +6466373538633537386164633464346536363733636234340a656162326537303433363632623737 +66313434653962386462653862613433633461316437306562356635363862666239306438393539 +6133636438643661660a306233343166616439313361643663316166306431396563663333373835 +3235