test_storage_volume_recover() {
    INCUS_IMPORT_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
    chmod +x "${INCUS_IMPORT_DIR}"
    spawn_incus "${INCUS_IMPORT_DIR}" true

    poolName=$(incus profile device get default root pool)
    poolDriver=$(incus storage show "${poolName}" | awk '/^driver:/ {print $2}')

    # Create custom block volume.
    incus storage volume create "${poolName}" vol1 --type=block

    # Import ISO.
    truncate -s 8MiB foo.iso
    incus storage volume import "${poolName}" ./foo.iso vol2 --type=iso

    # Delete database entry of the created custom block volume.
    incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='vol1'"
    incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='vol2'"

    # Ensure the custom block volume is no longer listed.
    ! incus storage volume show "${poolName}" vol1 || false
    ! incus storage volume show "${poolName}" vol2 || false

    if [ "$poolDriver" = "zfs" ]; then
        # Create filesystem volume.
        incus storage volume create "${poolName}" vol3

        # Create block_mode enabled volume.
        incus storage volume create "${poolName}" vol4 zfs.block_mode=true size=200MiB

        # Delete database entries of the created custom volumes.
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='vol3'"
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='vol4'"

        # Ensure the custom volumes are no longer listed.
        ! incus storage volume show "${poolName}" vol3 || false
        ! incus storage volume show "${poolName}" vol4 || false
    fi

    # Recover custom block volume.
    cat << EOF | incus admin recover
no
yes
yes
EOF

    # Ensure custom storage volume has been recovered.
    incus storage volume show "${poolName}" vol1 | grep -q 'content_type: block'
    incus storage volume show "${poolName}" vol2 | grep -q 'content_type: iso'

    if [ "$poolDriver" = "zfs" ]; then
        # Ensure custom storage volumes have been recovered.
        incus storage volume show "${poolName}" vol3 | grep -q 'content_type: filesystem'
        incus storage volume show "${poolName}" vol4 | grep -q 'content_type: filesystem'

        # Cleanup
        incus storage volume delete "${poolName}" vol3
        incus storage volume delete "${poolName}" vol4
    fi

    # Cleanup
    rm -f foo.iso
    incus storage volume delete "${poolName}" vol1
    incus storage volume delete "${poolName}" vol2
    shutdown_incus "${INCUS_IMPORT_DIR}"
}

test_container_recover() {
    INCUS_IMPORT_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
    chmod +x "${INCUS_IMPORT_DIR}"
    spawn_incus "${INCUS_IMPORT_DIR}" true

    (
        set -e

        # shellcheck disable=SC2030
        INCUS_DIR=${INCUS_IMPORT_DIR}
        incus_backend=$(storage_backend "$INCUS_DIR")

        ensure_import_testimage

        poolName=$(incus profile device get default root pool)
        poolDriver=$(incus storage show "${poolName}" | awk '/^driver:/ {print $2}')

        incus storage set "${poolName}" user.foo=bah
        incus project create test -c features.images=false -c features.profiles=true -c features.storage.volumes=true
        incus profile device add default root disk path=/ pool="${poolName}" --project test
        incus profile device add default eth0 nic nictype=p2p --project test
        incus project switch test

        # Basic no-op check.
        cat << EOF | incus admin recover | grep "No unknown storage pools or volumes found. Nothing to do."
no
yes
EOF

        # Recover container and custom volume that isn't mounted.
        incus init testimage c1
        incus storage volume create "${poolName}" vol1_test
        incus storage volume attach "${poolName}" vol1_test c1 /mnt
        incus start c1
        incus exec c1 --project test -- mount | grep /mnt
        echo "hello world" | incus exec c1 --project test -- tee /mnt/test.txt
        incus exec c1 --project test -- grep -xF "hello world" /mnt/test.txt
        incus stop -f c1
        incus snapshot create c1
        incus info c1

        incus storage volume snapshot create "${poolName}" vol1_test snap0
        incus storage volume show "${poolName}" vol1_test
        incus storage volume snapshot show "${poolName}" vol1_test/snap0

        # Remove container DB records and symlink.
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM instances WHERE name='c1'"
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='c1'"
        rm "${INCUS_DIR}/containers/test_c1"

        # Remove mount directories if block backed storage.
        if [ "$poolDriver" != "dir" ] && [ "$poolDriver" != "btrfs" ] && [ "$poolDriver" != "cephfs" ]; then
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/containers/test_c1"
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/containers-snapshots/test_c1/snap0"
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/containers-snapshots/test_c1"
        fi

        # Remove custom volume DB record.
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='vol1_test'"

        # Remove mount directories if block backed storage.
        if [ "$poolDriver" != "dir" ] && [ "$poolDriver" != "btrfs" ] && [ "$poolDriver" != "cephfs" ]; then
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/custom/test_vol1_test"
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/custom-snapshots/test_vol1_test/snap0"
            rmdir "${INCUS_DIR}/storage-pools/${poolName}/custom-snapshots/test_vol1_test"
        fi

        # Check container appears removed.
        ! ls "${INCUS_DIR}/containers/test_c1" || false
        ! incus info c1 || false
        ! incus storage volume show "${poolName}" container/c1 || false
        ! incus storage volume snapshot show "${poolName}" container/c1/snap0 || false

        if [ "$poolDriver" != "dir" ] && [ "$poolDriver" != "btrfs" ] && [ "$poolDriver" != "cephfs" ]; then
            ! ls "${INCUS_DIR}/storage-pools/${poolName}/containers/test_c1" || false
            ! ls "${INCUS_DIR}/storage-pools/${poolName}/containers-snapshots/test_c1" || false
        fi

        # Check custom volume appears removed.
        ! incus storage volume show "${poolName}" vol1_test || false
        ! incus storage volume snapshot show "${poolName}" vol1_test/snap0 || false

        # Shutdown Incus so pools are unmounted.
        shutdown_incus "${INCUS_DIR}"

        # Remove empty directory structures for pool drivers that don't have a mounted root.
        # This is so we can test the restoration of the storage pool directory structure.
        if [ "$poolDriver" != "dir" ] && [ "$poolDriver" != "btrfs" ] && [ "$poolDriver" != "cephfs" ]; then
            rm -rvf "${INCUS_DIR}/storage-pools/${poolName}"
        fi

        respawn_incus "${INCUS_DIR}" true

        cat << EOF | incus admin recover
no
yes
yes
EOF

        # Check container mount directories have been restored.
        ls "${INCUS_DIR}/containers/test_c1"
        ls "${INCUS_DIR}/storage-pools/${poolName}/containers/test_c1"
        ls "${INCUS_DIR}/storage-pools/${poolName}/containers-snapshots/test_c1/snap0"

        # Check custom volume mount directories have been restored.
        ls "${INCUS_DIR}/storage-pools/${poolName}/custom/test_vol1_test"
        ls "${INCUS_DIR}/storage-pools/${poolName}/custom-snapshots/test_vol1_test/snap0"

        # Check custom volume record exists with snapshot.
        incus storage volume show "${poolName}" vol1_test
        incus storage volume snapshot show "${poolName}" vol1_test/snap0

        # Check snapshot exists and container can be started.
        incus info c1 | grep snap0
        incus storage volume ls "${poolName}"
        incus storage volume show "${poolName}" container/c1
        incus storage volume snapshot show "${poolName}" container/c1/snap0
        incus start c1
        incus exec c1 --project test -- hostname

        # Check custom volume accessible.
        incus exec c1 --project test -- mount | grep /mnt
        incus exec c1 --project test -- grep -xF "hello world" /mnt/test.txt

        # Check snapshot can be restored.
        incus snapshot restore c1 snap0
        incus info c1
        incus exec c1 --project test -- hostname

        # Recover container that is running.
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM instances WHERE name='c1'"
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='c1'"

        # Restart Incus so internal mount counters are cleared for deleted (but running) container.
        shutdown_incus "${INCUS_DIR}"
        respawn_incus "${INCUS_DIR}" true

        cat << EOF | incus admin recover
no
yes
yes
EOF

        incus info c1 | grep snap0
        incus exec c1 --project test -- hostname
        incus snapshot restore c1 snap0
        incus info c1
        incus exec c1 --project test -- hostname

        # Test recover after pool DB config deletion too.
        poolConfigBefore=$(incus admin sql global "SELECT key,value FROM storage_pools_config JOIN storage_pools ON storage_pools.id = storage_pools_config.storage_pool_id WHERE storage_pools.name = '${poolName}' ORDER BY key")
        poolSource=$(incus storage get "${poolName}" source)
        poolExtraConfig=""

        case $poolDriver in
        lvm)
            poolExtraConfig="lvm.vg_name=$(incus storage get "${poolName}" lvm.vg_name)
"
            ;;
        zfs)
            poolExtraConfig="zfs.pool_name=$(incus storage get "${poolName}" zfs.pool_name)
"
            ;;
        ceph)
            poolExtraConfig="ceph.cluster_name=$(incus storage get "${poolName}" ceph.cluster_name)
ceph.osd.pool_name=$(incus storage get "${poolName}" ceph.osd.pool_name)
ceph.user.name=$(incus storage get "${poolName}" ceph.user.name)
"
            ;;
        linstor)
            poolExtraConfig="linstor.resource_group.place_count=$(incus storage get "${poolName}" linstor.resource_group.place_count)
linstor.volume.prefix=$(incus storage get "${poolName}" linstor.volume.prefix)
linstor.resource_group.name=$(incus storage get "${poolName}" linstor.resource_group.name)
"
            ;;
        truenas)
            poolExtraConfig="$(truenas_config)
$(truenas_allow_insecure)
$(truenas_api_key)
"
            ;;
        esac

        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM instances WHERE name='c1'"
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_volumes WHERE name='c1'"
        incus admin sql global "PRAGMA foreign_keys=ON; DELETE FROM storage_pools WHERE name='${poolName}'"

        cat << EOF | incus admin recover
yes
${poolName}
${poolDriver}
${poolSource}
${poolExtraConfig}
no
yes
yes
EOF

        # Check recovered pool config (from instance backup file) matches what originally was there.
        incus storage show "${poolName}"
        poolConfigAfter=$(incus admin sql global "SELECT key,value FROM storage_pools_config JOIN storage_pools ON storage_pools.id = storage_pools_config.storage_pool_id WHERE storage_pools.name = '${poolName}' ORDER BY key")
        echo "Before:"
        echo "${poolConfigBefore}"

        echo "After:"
        echo "${poolConfigAfter}"

        [ "${poolConfigBefore}" = "${poolConfigAfter}" ] || false
        incus storage show "${poolName}"

        incus info c1 | grep snap0
        incus exec c1 --project test -- ls
        incus snapshot restore c1 snap0
        incus info c1
        incus exec c1 --project test -- ls
        incus delete -f c1
        incus storage volume delete "${poolName}" vol1_test
        incus project switch default
        incus project delete test
    )

    # shellcheck disable=SC2031,2269
    INCUS_DIR=${INCUS_DIR}
    kill_incus "${INCUS_IMPORT_DIR}"
}

test_bucket_recover() {
    if ! command -v "minio" > /dev/null 2>&1; then
        echo "==> SKIP: Skip bucket recovery test due to missing minio"
        return
    fi

    (
        set -e

        poolName=$(incus profile device get default root pool)
        poolDriver=$(incus storage show "${poolName}" | awk '/^driver:/ {print $2}')
        bucketName="bucket123"

        # Skip ceph, linstor and truenas drivers, as they do not support storage buckets
        if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ] || [ "${poolDriver}" = "truenas" ]; then
            return 0
        fi

        # Create storage bucket
        incus storage bucket create "${poolName}" "${bucketName}"

        # Create storage bucket keys
        key1=$(incus storage bucket key create "${poolName}" "${bucketName}" key1 --role admin)
        key2=$(incus storage bucket key create "${poolName}" "${bucketName}" key2 --role read-only)
        key1_accessKey=$(echo "$key1" | awk '/^Access key/ { print $3 }')
        key1_secretKey=$(echo "$key1" | awk '/^Secret key/ { print $3 }')
        key2_accessKey=$(echo "$key2" | awk '/^Access key/ { print $3 }')
        key2_secretKey=$(echo "$key2" | awk '/^Secret key/ { print $3 }')

        # Remove bucket from global DB
        incus admin sql global "delete from storage_buckets where name = '${bucketName}'"

        # Recover bucket
        cat << EOF | incus admin recover
no
yes
yes
EOF

        # Verify bucket is recovered
        incus storage bucket ls "${poolName}" --format compact | grep "${bucketName}"

        # Verify bucket key with role admin is recovered
        recoveredKey1=$(incus storage bucket key show "${poolName}" "${bucketName}" "${key1_accessKey}")
        echo "${recoveredKey1}" | grep "role: admin"
        echo "${recoveredKey1}" | grep "access-key: ${key1_accessKey}"
        echo "${recoveredKey1}" | grep "secret-key: ${key1_secretKey}"

        # Verify bucket key with role read-only is recovered
        recoveredKey2=$(incus storage bucket key show "${poolName}" "${bucketName}" "${key2_accessKey}")
        echo "${recoveredKey2}" | grep "role: read-only"
        echo "${recoveredKey2}" | grep "access-key: ${key2_accessKey}"
        echo "${recoveredKey2}" | grep "secret-key: ${key2_secretKey}"
    )
}

test_backup_import() {
    test_backup_import_with_project
    test_backup_import_with_project fooproject
}

test_backup_import_with_project() {
    project="default"

    if [ "$#" -ne 0 ]; then
        # Create a projects
        project="$1"
        incus project create "$project"
        incus project create "$project-b"
        incus project switch "$project"

        deps/import-busybox --project "$project" --alias testimage
        deps/import-busybox --project "$project-b" --alias testimage

        # Add a root device to the default profile of the project
        pool="incustest-$(basename "${INCUS_DIR}")"
        incus profile device add default root disk path="/" pool="${pool}"
        incus profile device add default root disk path="/" pool="${pool}" --project "$project-b"
    fi

    ensure_import_testimage

    # shellcheck disable=2153
    ensure_has_localhost_remote "${INCUS_ADDR}"

    incus launch testimage c1
    incus launch testimage c2
    incus snapshot create c2

    incus_backend=$(storage_backend "$INCUS_DIR")

    # container only

    # create backup
    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus export c1 "${INCUS_DIR}/c1-optimized.tar.gz" --optimized-storage --instance-only
    fi

    incus export c1 "${INCUS_DIR}/c1.tar.gz" --instance-only
    incus delete --force c1

    # import backup, and ensure it's valid and runnable
    incus import "${INCUS_DIR}/c1.tar.gz"
    incus info c1
    incus start c1
    incus delete --force c1

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus import "${INCUS_DIR}/c1-optimized.tar.gz"
        incus info c1
        incus start c1
        incus delete --force c1
    fi

    # with snapshots

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus export c2 "${INCUS_DIR}/c2-optimized.tar.gz" --optimized-storage
    fi

    incus export c2 "${INCUS_DIR}/c2.tar.gz"
    incus delete --force c2

    incus import "${INCUS_DIR}/c2.tar.gz"
    incus import "${INCUS_DIR}/c2.tar.gz" c3
    incus info c2 | grep snap0
    incus info c3 | grep snap0
    incus start c2
    incus start c3
    incus stop c2 --force
    incus stop c3 --force

    if [ "$#" -ne 0 ]; then
        # Import into different project (before deleting earlier import).
        incus import "${INCUS_DIR}/c2.tar.gz" --project "$project-b"
        incus import "${INCUS_DIR}/c2.tar.gz" --project "$project-b" c3
        incus info c2 --project "$project-b" | grep snap0
        incus info c3 --project "$project-b" | grep snap0
        incus start c2 --project "$project-b"
        incus start c3 --project "$project-b"
        incus stop c2 --project "$project-b" --force
        incus stop c3 --project "$project-b" --force
        incus snapshot restore c2 snap0 --project "$project-b"
        incus snapshot restore c3 snap0 --project "$project-b"
        incus delete --force c2 --project "$project-b"
        incus delete --force c3 --project "$project-b"
    fi

    incus snapshot restore c2 snap0
    incus snapshot restore c3 snap0
    incus start c2
    incus start c3
    incus delete --force c2
    incus delete --force c3

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus import "${INCUS_DIR}/c2-optimized.tar.gz"
        incus import "${INCUS_DIR}/c2-optimized.tar.gz" c3
        incus info c2 | grep snap0
        incus info c3 | grep snap0
        incus start c2
        incus start c3
        incus stop c2 --force
        incus stop c3 --force
        incus snapshot restore c2 snap0
        incus snapshot restore c3 snap0
        incus start c2
        incus start c3
        incus delete --force c2
        incus delete --force c3
    fi

    # Test exporting container and snapshot names that container hyphens.
    # Also check that the container storage volume config is correctly captured and restored.
    default_pool="$(incus profile device get default root pool)"

    incus launch testimage c1-foo
    incus storage volume set "${default_pool}" container/c1-foo user.foo=c1-foo-snap0
    incus snapshot create c1-foo c1-foo-snap0
    incus storage volume set "${default_pool}" container/c1-foo user.foo=c1-foo-snap1
    incus snapshot create c1-foo c1-foo-snap1
    incus storage volume set "${default_pool}" container/c1-foo user.foo=post-c1-foo-snap1

    incus export c1-foo "${INCUS_DIR}/c1-foo.tar.gz"
    incus delete --force c1-foo

    incus import "${INCUS_DIR}/c1-foo.tar.gz"
    incus storage volume ls "${default_pool}"
    incus storage volume get "${default_pool}" container/c1-foo user.foo | grep -Fx "post-c1-foo-snap1"
    incus storage volume get "${default_pool}" container/c1-foo/c1-foo-snap0 user.foo | grep -Fx "c1-foo-snap0"
    incus storage volume get "${default_pool}" container/c1-foo/c1-foo-snap1 user.foo | grep -Fx "c1-foo-snap1"
    incus delete --force c1-foo

    # Create new storage pools
    incus storage create pool_1 dir
    incus storage create pool_2 dir

    # Export created container
    incus init testimage c3 -s pool_1
    incus export c3 "${INCUS_DIR}/c3.tar.gz"

    # Remove container and storage pool
    incus rm -f c3
    incus storage delete pool_1

    # This should succeed as it will fall back on the default pool
    incus import "${INCUS_DIR}/c3.tar.gz"

    incus rm -f c3

    # Remove root device
    incus profile device remove default root

    # This should fail as the expected storage is not available, and there is no default
    ! incus import "${INCUS_DIR}/c3.tar.gz" || false

    # Specify pool explicitly; this should fails as the pool doesn't exist
    ! incus import "${INCUS_DIR}/c3.tar.gz" -s pool_1 || false

    # Specify pool explicitly
    incus import "${INCUS_DIR}/c3.tar.gz" -s pool_2

    incus rm -f c3

    # Reset default storage pool
    incus profile device add default root disk path=/ pool="${default_pool}"

    incus storage delete pool_2

    if [ "$#" -ne 0 ]; then
        incus image rm testimage
        incus image rm testimage --project "$project-b"
        incus project switch default
        incus project delete "$project"
        incus project delete "$project-b"
    fi
}

test_backup_export() {
    test_backup_export_with_project
    test_backup_export_with_project fooproject
}

test_backup_export_with_project() {
    project="default"

    if [ "$#" -ne 0 ]; then
        # Create a project
        project="$1"
        incus project create "$project"
        incus project switch "$project"

        deps/import-busybox --project "$project" --alias testimage

        # Add a root device to the default profile of the project
        pool="incustest-$(basename "${INCUS_DIR}")"
        incus profile device add default root disk path="/" pool="${pool}"
    fi

    ensure_import_testimage
    ensure_has_localhost_remote "${INCUS_ADDR}"

    incus launch testimage c1
    incus snapshot create c1

    mkdir "${INCUS_DIR}/optimized" "${INCUS_DIR}/non-optimized"
    incus_backend=$(storage_backend "$INCUS_DIR")

    # container only

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus export c1 "${INCUS_DIR}/c1-optimized.tar.gz" --optimized-storage --instance-only
        tar -xzf "${INCUS_DIR}/c1-optimized.tar.gz" -C "${INCUS_DIR}/optimized"

        [ -f "${INCUS_DIR}/optimized/backup/index.yaml" ]
        [ -f "${INCUS_DIR}/optimized/backup/container.bin" ]
        [ ! -d "${INCUS_DIR}/optimized/backup/snapshots" ]
    fi

    incus export c1 "${INCUS_DIR}/c1.tar.gz" --instance-only
    tar -xzf "${INCUS_DIR}/c1.tar.gz" -C "${INCUS_DIR}/non-optimized"

    # check tarball content
    [ -f "${INCUS_DIR}/non-optimized/backup/index.yaml" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/container" ]
    [ ! -d "${INCUS_DIR}/non-optimized/backup/snapshots" ]

    rm -rf "${INCUS_DIR}/non-optimized/"* "${INCUS_DIR}/optimized/"*

    # with snapshots

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus export c1 "${INCUS_DIR}/c1-optimized.tar.gz" --optimized-storage
        tar -xzf "${INCUS_DIR}/c1-optimized.tar.gz" -C "${INCUS_DIR}/optimized"

        [ -f "${INCUS_DIR}/optimized/backup/index.yaml" ]
        [ -f "${INCUS_DIR}/optimized/backup/container.bin" ]
        [ -f "${INCUS_DIR}/optimized/backup/snapshots/snap0.bin" ]
    fi

    incus export c1 "${INCUS_DIR}/c1.tar.gz"
    tar -xzf "${INCUS_DIR}/c1.tar.gz" -C "${INCUS_DIR}/non-optimized"

    # check tarball content
    [ -f "${INCUS_DIR}/non-optimized/backup/index.yaml" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/container" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/snapshots/snap0" ]

    incus delete --force c1
    rm -rf "${INCUS_DIR}/optimized" "${INCUS_DIR}/non-optimized"

    # Check if hyphens cause issues when creating backups
    incus launch testimage c1-foo
    incus snapshot create c1-foo

    incus export c1-foo "${INCUS_DIR}/c1-foo.tar.gz"

    incus delete --force c1-foo

    if [ "$#" -ne 0 ]; then
        incus image rm testimage
        incus project switch default
        incus project delete "$project"
    fi
}

test_backup_rename() {
    ensure_import_testimage
    ensure_has_localhost_remote "${INCUS_ADDR}"

    if ! incus query -X POST /1.0/instances/c1/backups/backupmissing -d '{\"name\": \"backupnewname\"}' --wait 2>&1 | grep -q "Error: Instance backup not found"; then
        echo "invalid rename response for missing container"
        false
    fi

    incus init testimage c1

    if ! incus query -X POST /1.0/instances/c1/backups/backupmissing -d '{\"name\": \"backupnewname\"}' --wait 2>&1 | grep -q "Error: Instance backup not found"; then
        echo "invalid rename response for missing backup"
        false
    fi

    # Create backup
    incus query -X POST --wait -d '{\"name\":\"foo\"}' /1.0/instances/c1/backups

    # All backups should be listed
    incus query /1.0/instances/c1/backups | jq .'[0]' | grep instances/c1/backups/foo

    # The specific backup should exist
    incus query /1.0/instances/c1/backups/foo

    # Rename the container which should rename the backup(s) as well
    incus mv c1 c2

    # All backups should be listed
    incus query /1.0/instances/c2/backups | jq .'[0]' | grep instances/c2/backups/foo

    # The specific backup should exist
    incus query /1.0/instances/c2/backups/foo

    # The old backup should not exist
    ! incus query /1.0/instances/c1/backups/foo || false

    incus delete --force c2
}

test_backup_volume_export() {
    test_backup_volume_export_with_project default "incustest-$(basename "${INCUS_DIR}")"
    test_backup_volume_export_with_project fooproject "incustest-$(basename "${INCUS_DIR}")"

    if [ "$incus_backend" = "ceph" ] && [ -n "${INCUS_CEPH_CEPHFS:-}" ]; then
        custom_vol_pool="incustest-$(basename "${INCUS_DIR}")-cephfs"
        incus storage create "${custom_vol_pool}" cephfs source="${INCUS_CEPH_CEPHFS}/$(basename "${INCUS_DIR}")-cephfs"

        test_backup_volume_export_with_project default "${custom_vol_pool}"
        test_backup_volume_export_with_project fooproject "${custom_vol_pool}"

        incus storage rm "${custom_vol_pool}"
    fi
}

test_backup_volume_export_with_project() {
    pool="incustest-$(basename "${INCUS_DIR}")"
    project="$1"
    custom_vol_pool="$2"

    if [ "${project}" != "default" ]; then
        # Create a project.
        incus project create "$project"
        incus project create "$project-b"
        incus project switch "$project"

        deps/import-busybox --project "$project" --alias testimage
        deps/import-busybox --project "$project-b" --alias testimage

        # Add a root device to the default profile of the project.
        incus profile device add default root disk path="/" pool="${pool}"
    fi

    ensure_import_testimage
    ensure_has_localhost_remote "${INCUS_ADDR}"

    mkdir "${INCUS_DIR}/optimized" "${INCUS_DIR}/non-optimized"
    incus_backend=$(storage_backend "$INCUS_DIR")

    # Create test container.
    incus init testimage c1

    # Create custom storage volume.
    incus storage volume create "${custom_vol_pool}" testvol

    # Attach storage volume to the test container and start.
    incus storage volume attach "${custom_vol_pool}" testvol c1 /mnt
    incus start c1

    # Create file on the custom volume.
    echo foo | incus file push - c1/mnt/test

    # Snapshot the custom volume.
    incus storage volume set "${custom_vol_pool}" testvol user.foo=test-snap0
    incus storage volume snapshot create "${custom_vol_pool}" testvol test-snap0

    # Change the content (the snapshot will contain the old value).
    echo bar | incus file push - c1/mnt/test

    incus storage volume set "${custom_vol_pool}" testvol user.foo=test-snap1
    incus storage volume snapshot create "${custom_vol_pool}" testvol test-snap1
    incus storage volume set "${custom_vol_pool}" testvol user.foo=post-test-snap1

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        # Create optimized backup without snapshots.
        incus storage volume export "${custom_vol_pool}" testvol "${INCUS_DIR}/testvol-optimized.tar.gz" --volume-only --optimized-storage

        [ -f "${INCUS_DIR}/testvol-optimized.tar.gz" ]

        # Extract backup tarball.
        tar -xzf "${INCUS_DIR}/testvol-optimized.tar.gz" -C "${INCUS_DIR}/optimized"

        [ -f "${INCUS_DIR}/optimized/backup/index.yaml" ]
        [ -f "${INCUS_DIR}/optimized/backup/volume.bin" ]
        [ ! -d "${INCUS_DIR}/optimized/backup/volume-snapshots" ]
    fi

    # Create non-optimized backup without snapshots.
    incus storage volume export "${custom_vol_pool}" testvol "${INCUS_DIR}/testvol.tar.gz" --volume-only

    [ -f "${INCUS_DIR}/testvol.tar.gz" ]

    # Extract non-optimized backup tarball.
    tar -xzf "${INCUS_DIR}/testvol.tar.gz" -C "${INCUS_DIR}/non-optimized"

    # Check tarball content.
    [ -f "${INCUS_DIR}/non-optimized/backup/index.yaml" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/volume" ]
    [ "$(cat "${INCUS_DIR}/non-optimized/backup/volume/test")" = "bar" ]
    [ ! -d "${INCUS_DIR}/non-optimized/backup/volume-snapshots" ]

    ! grep -q -- '- test-snap0' "${INCUS_DIR}/non-optimized/backup/index.yaml" || false

    rm -rf "${INCUS_DIR}/non-optimized/"*
    rm "${INCUS_DIR}/testvol.tar.gz"

    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        # Create optimized backup with snapshots.
        incus storage volume export "${custom_vol_pool}" testvol "${INCUS_DIR}/testvol-optimized.tar.gz" --optimized-storage

        [ -f "${INCUS_DIR}/testvol-optimized.tar.gz" ]

        # Extract backup tarball.
        tar -xzf "${INCUS_DIR}/testvol-optimized.tar.gz" -C "${INCUS_DIR}/optimized"

        [ -f "${INCUS_DIR}/optimized/backup/index.yaml" ]
        [ -f "${INCUS_DIR}/optimized/backup/volume.bin" ]
        [ -f "${INCUS_DIR}/optimized/backup/volume-snapshots/test-snap0.bin" ]
    fi

    # Create non-optimized backup with snapshots.
    incus storage volume export "${custom_vol_pool}" testvol "${INCUS_DIR}/testvol.tar.gz"

    [ -f "${INCUS_DIR}/testvol.tar.gz" ]

    # Extract backup tarball.
    tar -xzf "${INCUS_DIR}/testvol.tar.gz" -C "${INCUS_DIR}/non-optimized"

    # Check tarball content.
    [ -f "${INCUS_DIR}/non-optimized/backup/index.yaml" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/volume" ]
    [ "$(cat "${INCUS_DIR}/non-optimized/backup/volume/test")" = "bar" ]
    [ -d "${INCUS_DIR}/non-optimized/backup/volume-snapshots/test-snap0" ]
    [ "$(cat "${INCUS_DIR}/non-optimized/backup/volume-snapshots/test-snap0/test")" = "foo" ]

    grep -q -- '- test-snap0' "${INCUS_DIR}/non-optimized/backup/index.yaml"

    rm -rf "${INCUS_DIR}/non-optimized/"*

    # Test non-optimized import.
    incus stop -f c1
    incus storage volume detach "${custom_vol_pool}" testvol c1
    incus storage volume delete "${custom_vol_pool}" testvol
    incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol.tar.gz"
    incus storage volume ls "${custom_vol_pool}"
    incus storage volume get "${custom_vol_pool}" testvol user.foo | grep -Fx "post-test-snap1"
    incus storage volume snapshot show "${custom_vol_pool}" testvol/test-snap0
    incus storage volume get "${custom_vol_pool}" testvol/test-snap0 user.foo | grep -Fx "test-snap0"
    incus storage volume get "${custom_vol_pool}" testvol/test-snap1 user.foo | grep -Fx "test-snap1"

    incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol.tar.gz" testvol2
    incus storage volume attach "${custom_vol_pool}" testvol c1 /mnt
    incus storage volume attach "${custom_vol_pool}" testvol2 c1 /mnt2
    incus start c1
    incus exec c1 --project "$project" -- stat /mnt/test
    incus exec c1 --project "$project" -- stat /mnt2/test
    incus stop -f c1

    if [ "${project}" != "default" ]; then
        # Import into different project (before deleting earlier import).
        incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol.tar.gz" --project "$project-b"
        incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol.tar.gz" --project "$project-b" testvol2
        incus storage volume delete "${custom_vol_pool}" testvol --project "$project-b"
        incus storage volume delete "${custom_vol_pool}" testvol2 --project "$project-b"
    fi

    # Test optimized import.
    if [ "$incus_backend" = "btrfs" ] || [ "$incus_backend" = "zfs" ]; then
        incus storage volume detach "${custom_vol_pool}" testvol c1
        incus storage volume detach "${custom_vol_pool}" testvol2 c1
        incus storage volume delete "${custom_vol_pool}" testvol
        incus storage volume delete "${custom_vol_pool}" testvol2
        incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol-optimized.tar.gz"
        incus storage volume ls "${custom_vol_pool}"
        incus storage volume get "${custom_vol_pool}" testvol user.foo | grep -Fx "post-test-snap1"
        incus storage volume get "${custom_vol_pool}" testvol/test-snap0 user.foo | grep -Fx "test-snap0"
        incus storage volume get "${custom_vol_pool}" testvol/test-snap1 user.foo | grep -Fx "test-snap1"

        incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol-optimized.tar.gz" testvol2
        incus storage volume attach "${custom_vol_pool}" testvol c1 /mnt
        incus storage volume attach "${custom_vol_pool}" testvol2 c1 /mnt2
        incus start c1
        incus exec c1 --project "$project" -- stat /mnt/test
        incus exec c1 --project "$project" -- stat /mnt2/test
        incus stop -f c1

        if [ "${project}" != "default" ]; then
            # Import into different project (before deleting earlier import).
            incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol-optimized.tar.gz" --project "$project-b"
            incus storage volume import "${custom_vol_pool}" "${INCUS_DIR}/testvol-optimized.tar.gz" --project "$project-b" testvol2
            incus storage volume delete "${custom_vol_pool}" testvol --project "$project-b"
            incus storage volume delete "${custom_vol_pool}" testvol2 --project "$project-b"
        fi
    fi

    # Clean up.
    rm -rf "${INCUS_DIR}/non-optimized/"* "${INCUS_DIR}/optimized/"*
    incus storage volume detach "${custom_vol_pool}" testvol c1
    incus storage volume detach "${custom_vol_pool}" testvol2 c1
    incus storage volume rm "${custom_vol_pool}" testvol
    incus storage volume rm "${custom_vol_pool}" testvol2
    incus rm -f c1
    rmdir "${INCUS_DIR}/optimized"
    rmdir "${INCUS_DIR}/non-optimized"

    if [ "${project}" != "default" ]; then
        incus project switch default
        incus image rm testimage --project "$project"
        incus image rm testimage --project "$project-b"
        incus project delete "$project"
        incus project delete "$project-b"
    fi
}

test_backup_volume_rename_delete() {
    ensure_has_localhost_remote "${INCUS_ADDR}"

    pool="incustest-$(basename "${INCUS_DIR}")"

    # Create test volume.
    incus storage volume create "${pool}" vol1

    if ! incus query -X POST /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups/backupmissing -d '{\"name\": \"backupnewname\"}' --wait 2>&1 | grep -q "Error: Storage volume backup not found"; then
        echo "invalid rename response for missing storage volume"
        false
    fi

    # Create backup.
    incus query -X POST --wait -d '{\"name\":\"foo\"}' /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups

    # All backups should be listed.
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups | jq .'[0]' | grep storage-pools/"${pool}"/volumes/custom/vol1/backups/foo

    # The specific backup should exist.
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups/foo
    stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol1/foo

    # Delete backup and check it is removed from DB and disk.
    incus query -X DELETE --wait /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups/foo
    ! incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups/foo || false
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol1/foo || false
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol1 || false

    # Create backup again to test rename.
    incus query -X POST --wait -d '{\"name\":\"foo\"}' /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups

    # Rename the container which should rename the backup(s) as well.
    incus storage volume rename "${pool}" vol1 vol2

    # All backups should be listed.
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol2/backups | jq .'[0]' | grep storage-pools/"${pool}"/volumes/custom/vol2/backups/foo

    # The specific backup should exist.
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol2/backups/foo
    stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol2/foo

    # The old backup should not exist.
    ! incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol1/backups/foo || false
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol1/foo || false
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol1 || false

    # Rename backup itself and check its renamed in DB and on disk.
    incus query -X POST --wait -d '{\"name\":\"foo2\"}' /1.0/storage-pools/"${pool}"/volumes/custom/vol2/backups/foo
    incus query /1.0/storage-pools/"${pool}"/volumes/custom/vol2/backups | jq .'[0]' | grep storage-pools/"${pool}"/volumes/custom/vol2/backups/foo2
    stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol2/foo2
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol2/foo || false

    # Remove volume and check the backups are removed too.
    incus storage volume rm "${pool}" vol2
    ! stat "${INCUS_DIR}"/backups/custom/"${pool}"/default_vol2 || false
}

test_backup_different_instance_uuid() {
    ensure_import_testimage
    ensure_has_localhost_remote "${INCUS_ADDR}"

    echo "==> Checking instances UUID during backup operation"
    incus launch testimage c1
    initialUUID=$(incus config get c1 volatile.uuid)
    initialGenerationID=$(incus config get c1 volatile.uuid.generation)

    # export and import to trigger new UUID generation
    incus export c1 "${INCUS_DIR}/c1.tar.gz"
    incus delete -f c1
    incus import "${INCUS_DIR}/c1.tar.gz"

    newUUID=$(incus config get c1 volatile.uuid)
    newGenerationID=$(incus config get c1 volatile.uuid.generation)

    if [ "${initialGenerationID}" != "${newGenerationID}" ] || [ "${initialUUID}" != "${newUUID}" ]; then
        echo "==> UUID of the instance should remain the same after importing the backup file"
        false
    fi

    incus delete -f c1
}

test_backup_volume_expiry() {
    poolName=$(incus profile device get default root pool)

    # Create custom volume.
    incus storage volume create "${poolName}" vol1

    # Create storage volume backups using the API directly.
    # The first one is created with an expiry date, the second one never expires.
    incus query -X POST -d '{\"expires_at\":\"2023-07-17T00:00:00Z\"}' /1.0/storage-pools/"${poolName}"/volumes/custom/vol1/backups
    incus query -X POST -d '{}' /1.0/storage-pools/"${poolName}"/volumes/custom/vol1/backups

    # Check that both backups are listed.
    [ "$(incus query /1.0/storage-pools/"${poolName}"/volumes/custom/vol1/backups | jq '.[]' | wc -l)" -eq 2 ]

    # Restart Incus which will trigger the task which removes expired volume backups.
    shutdown_incus "${INCUS_DIR}"
    respawn_incus "${INCUS_DIR}" true

    # Check that there's only one backup remaining.
    [ "$(incus query /1.0/storage-pools/"${poolName}"/volumes/custom/vol1/backups | jq '.[]' | wc -l)" -eq 1 ]

    # Cleanup.
    incus storage volume delete "${poolName}" vol1
}

test_backup_export_import_recover() {
    (
        set -e

        poolName=$(incus profile device get default root pool)

        ensure_import_testimage
        ensure_has_localhost_remote "${INCUS_ADDR}"

        # Create and export an instance.
        incus launch testimage c1
        incus export c1 "${INCUS_DIR}/c1.tar.gz"
        incus delete -f c1

        # Import instance and remove no longer required tarball.
        incus import "${INCUS_DIR}/c1.tar.gz" c2
        rm "${INCUS_DIR}/c1.tar.gz"

        # Remove imported instance entries from database.
        incus admin sql global "delete from instances where name = 'c2'"
        incus admin sql global "delete from storage_volumes where name = 'c2'"

        # Recover removed instance.
        cat << EOF | incus admin recover
no
yes
yes
EOF

        # Remove recovered instance.
        incus rm -f c2
    )
}

test_backup_export_import_instance_only() {
    poolName=$(incus profile device get default root pool)

    ensure_import_testimage
    ensure_has_localhost_remote "${INCUS_ADDR}"

    # Create an instance with snapshot.
    incus init testimage c1
    incus snapshot create c1

    # Export the instance and remove it.
    incus export c1 "${INCUS_DIR}/c1.tar.gz" --instance-only
    incus delete -f c1

    # Import the instance from tarball.
    incus import "${INCUS_DIR}/c1.tar.gz"

    # Verify imported instance has no snapshots.
    [ "$(incus query "/1.0/storage-pools/${poolName}/volumes/container/c1/snapshots" | jq "length == 0")" = "true" ]

    rm "${INCUS_DIR}/c1.tar.gz"
    incus delete -f c1
}
