chore: update terraform outputs

.gitea/workflows/deploy.yml

@@ -1,6 +1,8 @@
 name: Deploy VM and App
 
 on:
+  push:
+
   workflow_dispatch:
     inputs:
       tfvars_file:
@@ -13,57 +15,64 @@ on:
           - multi.tfvars.example
 
 jobs:
-  terraform-deploy:
+  terraform-ansible-deploy:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: terraform
     steps:
       - uses: actions/checkout@v4
 
+      - name: Check files & Select tfvars
+        shell: bash
+        run: |
+          #rm ansible/inventory/inventory.yml
+          mkdir -p ansible/inventory
+          cd terraform
+          #rm vm_data.yml
+          cp "${{ inputs.tfvars_file || 'single.tfvars.example' }}" terraform.tfvars
+
       - uses: hashicorp/setup-terraform@v4
 
-      - name: Select tfvars
+      - name: Check path
-        run: cp "${{ inputs.tfvars_file }}" terraform.tfvars
+        run: pwd
 
       - name: Terraform init
         run: terraform init
+        working-directory: "terraform"
 
       - name: Terraform apply
         run: terraform apply -auto-approve
-
+        working-directory: "terraform"
-      - name: Write inventory
-        run: |
-          mkdir -p ../ansible/inventory
-          terraform output -json vm_ipv4_addresses | jq -r '
-            to_entries[] | "[app]\n\(.value) ansible_user=cloud"
-          ' > ../ansible/inventory/hosts.ini
-
-      - name: Write tags
-        run: terraform output -json vm_tags > ../ansible/vm_tags.json
-
-  ansible-configure:
-    needs: terraform
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ansible
-    steps:
-      - uses: actions/checkout@v4
 
       - name: Install Ansible
         shell: bash
         run: |
           sudo apt-get update
-          sudo apt-get install -y ansible
+          sudo apt-get install -y ansible rsync
 
       - name: Set up SSH
         run: |
           mkdir -p ~/.ssh
-          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/vlans_rsa
-          chmod 600 ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/vlans_rsa
-          echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null\n" > ~/.ssh/config
+          cat > ~/.ssh/config <<'EOF'
+          Host *
+            StrictHostKeyChecking no
+            UserKnownHostsFile=/dev/null
+          EOF
 
       - name: Run playbook
         run: |
-          ansible-playbook playbooks/docker_copy.yml -i inventory/inventory.yml -u cloud --private-key ~/.ssh/id_rsa
+          ansible-playbook ansible/playbooks/docker_copy.yml -i ansible/inventory/inventory.yml -u cloud --private-key ~/.ssh/vlans_rsa
+
+      - name: Configure Git
+        run: |
+          git config user.name "git-bot"
+          git config user.email "got-bot@text.com"
+
+      - name: Commit and push to Gitea
+        run: |
+          git remote set-url origin https://$GITEA_USERNAME:${{ secrets.GIT_BOT_TOKEN }}@tea.charcarservices.uk/CC/N8N.git
+          git add terraform/vm_data.yml ansible/inventory/inventory.yml
+          git diff --cached --quiet || git commit -m "chore: update terraform outputs"
+          git push origin HEAD:main
+        env:
+          GITEA_USERNAME: git-bot # or your bot account

ansible/inventory/inventory.yml

@@ -0,0 +1,8 @@
+"all":
+  "children":
+    "n8n":
+      "hosts":
+        "n8n-01":
+          "ansible_host": "192.168.10.107"
+  "vars":
+    "ansible_user": "cloud"

terraform/main.tf

@@ -9,10 +9,10 @@ locals {
       service_name = v.service_name
       vm_name      = v.vm_name
       node_name    = v.node_name
-      ipv4_address = module.vm[k].vm_ipv4_address
+      ipv4_address = module.vm-n8n[k].vm_ipv4_address
       vm_tags      = concat(
         try(v.vm_tags, []),
-        ["terraform", "docker", v.service_name, "ip-${replace(module.vm[k].vm_ipv4_address, ".", "-")}"]
+        ["terraform", "docker", v.service_name, "ip-${replace(module.vm-n8n[k].vm_ipv4_address, ".", "-")}"]
       )
     }
   }
@@ -20,7 +20,7 @@ locals {
 
 module "vm-n8n" {
   for_each = local.instance_map
-  source   = "./modules/proxmox_ubuntu_cloudinit_template"
+  source   = "./modules/proxmox_ubuntu_cloudinit_clone"
 
   vm_name        = each.value.vm_name
   node_name      = each.value.node_name
@@ -40,12 +40,12 @@ module "vm-n8n" {
 
 module "inventory" {
   source   = "./modules/proxmox_ansible_inventory"
-  filename = "${path.module}/ansible/inventory/inventory.yml"
+  filename = "${abspath("${path.root}/..")}/ansible/inventory/inventory.yml"
   instances = local.vm_created
 }
 
 module "vm_data" {
   source    = "./modules/proxmox_vm_data"
-  filename  = "${path.module}/terraform/vm_data.yml"
+  filename  = "${abspath("${path.root}/..")}/terraform/vm_data.yml"
   instances = local.vm_created
 }

terraform/modules/proxmox_ubuntu_cloudinit_clone/main.tf

@@ -0,0 +1,123 @@
+locals {
+  selected_instance = one([
+    for cfg in var.instance_configs :
+    cfg if cfg.crispy_name == var.node_name
+  ])
+}
+
+resource "proxmox_virtual_environment_file" "cloud_config" {
+  content_type = "snippets"
+  datastore_id = "local"
+  node_name    = "pop"
+
+  source_raw {
+    file_name = "vm.cloud-config.yaml" # The name of the snippet file
+    data      = <<-EOF
+    #cloud-config
+    hostname: ${var.vm_name}
+
+    package_update: true
+    package_upgrade: true
+
+    system_info:
+      default_user:
+        groups: [ docker ]
+
+    users:
+      - default
+      - name: cloud
+        groups: 
+          - sudo
+          - docker
+        shell: /bin/bash
+        ssh-authorized-keys:
+          - "${var.vm_user_sshkey}" # Inject user's SSH key
+        sudo: ALL=(ALL) NOPASSWD:ALL
+
+    packages:
+      - qemu-guest-agent   
+      - apt-transport-https
+      - ca-certificates
+      - curl
+      - gnupg
+      - lsb-release
+      - unattended-upgrades 
+
+    runcmd:
+      - systemctl enable qemu-guest-agent 
+      - mkdir -p /etc/apt/keyrings
+      - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+      - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+      - apt-get update
+      - apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+      - systemctl enable docker
+      - systemctl start docker
+      - reboot                     
+    EOF
+  }
+}
+
+resource "proxmox_virtual_environment_vm" "ubuntu_22_minimal_clone" {
+  name      = var.vm_name           # VM name
+  node_name = var.node_name         # Proxmox node to deploy the VM
+  tags      = var.vm_tags           # Optional VM tags for categorization
+
+  agent {
+    enabled = true                  # Enable the QEMU guest agent
+  }
+
+  stop_on_destroy = true            # Ensure VM is stopped gracefully when destroyed
+
+
+  clone {
+    vm_id     = local.selected_instance.vmid    # ID of the source template
+    node_name = local.selected_instance.crispy_name # Node of the source template
+  }
+
+  bios    = var.vm_bios             # BIOS type (e.g., seabios or ovmf)
+  machine = var.vm_machine          # Machine type (e.g., q35)
+
+  cpu {
+    cores = var.vm_cpu              # Number of CPU cores
+    type  = "host"                  # Use host CPU type for best compatibility/performance
+  }
+
+  memory {
+    dedicated = var.vm_ram          # RAM in MB
+  }
+
+  disk {
+    datastore_id = var.node_datastore # Datastore to hold the disk
+    interface    = "scsi0"             # Primary disk interface
+    size         = var.vm_size      
+  }
+
+
+  initialization {
+    user_data_file_id = proxmox_virtual_environment_file.cloud_config.id # Link the cloud-init file
+    datastore_id      = var.node_datastore
+    interface         = "scsi1"             # Separate interface for cloud-init
+    ip_config {
+      ipv4 {
+        address = "dhcp"            # Get IP via DHCP
+      }
+    }
+  }
+
+  network_device {
+    bridge  = var.bridge               # Use the default bridge 
+  }
+
+  lifecycle {
+    ignore_changes = [              # Ignore initialization section after first depoloyment for idempotency
+      initialization
+    ]
+  }
+}
+
+
+
+
+
+
+

terraform/modules/proxmox_ubuntu_cloudinit_clone/output.tf

@@ -0,0 +1,3 @@
+output "vm_ipv4_address" {
+  value = proxmox_virtual_environment_vm.ubuntu_22_minimal_clone.ipv4_addresses[1][0]
+}

terraform/modules/proxmox_ubuntu_cloudinit_clone/providers.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "0.106.0"
+     #url = https://registry.terraform.io/providers/bpg/proxmox/latest/docs/guides/clone-vm 
+    }
+    aws = {
+      source  = "hashicorp/aws"
+      version = "6.38.0"
+    }
+  }
+}

terraform/modules/proxmox_ubuntu_cloudinit_clone/variables.tf

@@ -0,0 +1,140 @@
+variable "pm_api_url" { default = "https://192.168.10.201:8006/api2/json" }
+variable "pm_api_token" { default = "terraform@pve!provider=760580c4-5c1f-462b-986a-dd244c6c95f2" }
+
+variable "storage"      { default = "hlst" }
+variable "bridge"       { default = "vmbr0" }
+
+variable "os_type" {
+  default = "alpine"
+}
+
+variable "instance_configs" {
+  type = list(object({
+    crispy_name = string
+    vmid           = string
+  }))
+  default = [
+    { crispy_name = "snap", vmid = "9002" },
+    { crispy_name = "crackle", vmid = "9000" },
+    { crispy_name = "pop", vmid = "900" }
+  ]
+}
+
+variable "clone_count" {
+  type    = number
+  default = 1
+}
+
+variable "vm_count"    { default = 1 }
+variable "name_prefix" { default = "dev" }
+variable "vm_ram"      { default = 2048 }
+variable "vm_cpu"       { default = 1 }
+variable "vm_size"      { default = 10 }
+variable "vm_bios" {
+  description = "Type of BIOS used for the VM"
+  type        = string
+  default     = "ovmf"
+}
+
+variable "vm_machine" {
+  description = "Type of machine used for the VM"
+  type        = string
+  default     = "q35"
+}
+
+variable "vm_tags" {
+  description = "Tags for the VM"
+  type        = list(any)
+  default     = ["test", "terraform"]
+}
+
+variable "ipconfig0"       { default = "ip=dhcp" }
+
+variable "access_key" {
+  type        = string
+  default = "GK242d456c0692a9d4cc102206"
+}
+
+variable "secret_key" {
+  type        = string
+  default = "1d7e22b7a8892cb11b569017659aa511b37b53287c4d1699c310d9f8ac76df09"
+}
+
+variable "region" {
+  type        = string
+  default = "garage"
+}
+
+variable "endpoints_s3" {
+  type        = string
+  description = "S3 endpoint"
+  default = "http://192.168.10.109:3909"
+}
+
+variable "skip_credentials_validation" {
+  type        = bool
+  default     = true
+}
+
+variable "skip_requesting_account_id" {
+  type        = bool
+  default     = true
+}
+
+variable "skip_metadata_api_check" {
+  type        = bool
+  default     = true
+}
+
+variable "skip_region_validation" {
+  type        = bool
+  default     = true
+}
+
+variable "use_path_style" {
+  type        = bool
+  default     = true
+}
+
+variable "use_lockfile" {
+  type        = bool
+  default     = true
+}
+
+variable "vm_name" {
+  description = "Hostname of the VM"
+  type        = string
+  default     = "Lab"
+}
+
+variable "vm_user_sshkey" {
+  description = "Admin user SSH key of the VM"
+  type        = string
+  default     = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRlWaLBt/qmWY01Cd6jN/YxLnlT+6lg+evEdN/dIajirdTj1rCbAdlG3WYvo+4BpN17HK3/eGQpGUMbgI/8MVd8YPODcD34gaNX0w2v66BwHx+S6BZUpz5T2IoQT0JtSv/TtFICoff5gXdNRpfd4eWsmTioEqLA6oToJLE4dn3jvAzFi9y7fyLqvuoQMmPidYYJjGT30eiULtXNspoEP+GmuWmVEu+znzMWaKDWKdOsii4Cv1aWCRKSDDRzDBrZI2mP+Vm4HDQBdgDYRw4ehumMDtfaSjyJCnrk691bIM+wxzICuIEecg5kq5HcUPvo2mFyWPAEXb5xlXnuopYEBd7 Generated By NeoServer"
+}
+
+
+variable "node_name" {
+  description = "Proxmox host for the VM"
+  type        = string
+  default     = "pop"
+}
+
+variable "node_datastore" {
+  description = "Datastore used for VM storage"
+  type        = string
+  default     = "hlst"
+}
+
+variable "vm_template" {
+  description = "Template of the VM"
+  type        = string
+  default     = "ubuntu-cloud"
+}
+
+
+variable "vm_user" {
+  description = "Admin user of the VM"
+  type        = string
+  default     = "cloud"
+}

terraform/output.tf

@@ -1,20 +1,41 @@
 output "vm_ipv4_addresses" {
+  description = "Map of instance key to VM IPv4 address"
   value = {
     for k, m in module.vm-n8n : k => m.vm_ipv4_address
   }
 }
 
-output "vm_tags" {
+output "service_names" {
+  description = "Map of instance key to service name"
   value = {
-    for k, m in module.vm-n8n : k => concat(
+    for k, v in local.instance_map : k => v.service_name
-      try(local.instance_map[k].vm_tags, []),
-      ["terraform", "docker", local.instance_map[k].service_name, "ip-${replace(m.vm_ipv4_address, ".", "-")}"]
-    )
   }
 }
 
-output "service_names" {
+output "vm_names" {
+  description = "Map of instance key to VM name"
   value = {
-    for k, cfg in local.instance_map : k => cfg.service_name
+    for k, v in local.instance_map : k => v.vm_name
   }
 }
+
+output "vm_tag_data" {
+  description = "Map of instance key to node, VM name, and final tags"
+  value = {
+    for k, v in local.vm_created : k => {
+      node_name = v.node_name
+      vm_name   = v.vm_name
+      tags      = v.vm_tags
+    }
+  }
+}
+
+output "inventory_file" {
+  description = "Path to the generated Ansible inventory"
+  value       = module.inventory.filename
+}
+
+output "vm_data_file" {
+  description = "Path to the generated VM data file"
+  value       = module.vm_data.filename
+}

terraform/variables.tf

@@ -1,40 +1,44 @@
 variable "instance_mode" {
-  type    = string
+  type        = string
-  default = "single"
+  description = "single or multiple"
+
+  validation {
+    condition     = contains(["single", "multiple"], var.instance_mode)
+    error_message = "instance_mode must be either single or multiple."
+  }
 }
 
 variable "instance" {
-  description = "Single instance definition"
   type = object({
-    service_name   = string
+    service_name = string
-    vm_name        = string
+    vm_name      = string
-    node_name      = string
+    node_name    = string
-    vm_cpu         = optional(number)
+    vm_cpu       = optional(number)
-    vm_ram         = optional(number)
+    vm_ram       = optional(number)
-    vm_size        = optional(string)
+    vm_size      = optional(string)
-    app_port       = number
+    vm_tags      = optional(list(string))
-    app_image      = string
-    vm_tags        = optional(list(string))
   })
-  default = null
+  default   = null
+  nullable  = true
+  description = "Used only when instance_mode = single."
 }
 
 variable "instances" {
-  description = "Multiple instance definitions"
   type = map(object({
-    service_name   = string
+    service_name = string
-    vm_name        = string
+    vm_name      = string
-    node_name      = string
+    node_name    = string
-    vm_cpu         = optional(number)
+    vm_cpu       = optional(number)
-    vm_ram         = optional(number)
+    vm_ram       = optional(number)
-    vm_size        = optional(string)
+    vm_size      = optional(string)
-    app_port       = number
+    vm_tags      = optional(list(string))
-    app_image      = string
-    vm_tags        = optional(list(string))
   }))
-  default = {}
+  default     = {}
+  nullable    = false
+  description = "Used only when instance_mode = multiple."
 }
 
+
 variable "vm_defaults" {
   type = object({
     node_datastore = string

terraform/vm_data.yml

@@ -0,0 +1,10 @@
+"vm_tag_data":
+  "main":
+    "node_name": "pop"
+    "tags":
+    - "agentic"
+    - "terraform"
+    - "docker"
+    - "n8n"
+    - "ip-192-168-10-107"
+    "vm_name": "n8n-01"