Taskfile.yml

# This Taskfile makes a nightly mirror of a Dropbox account in one Proton Drive folder.
# An external scheduler starts sync.yml, and GitHub Actions runs it. Each run compares the
# Dropbox listing with the state in R2. It uploads the changed files in batches until its
# time budget ends, and the next run continues. The state is an age-encrypted SQLite
# database, and only R2 has the data about how far the mirror got.
#
# The toolbox and the proton engine are parts of katoptra/lib. This file includes the two
# at v2, with `flatten: true`. The README of lib gives the items that each one supplies.
# The items in this file are for this mirror only:
#   - pipeline and plan-pipeline: the steps in sequence, one migrator command for each step,
#     with the clock, report and ping of the toolbox
#   - the operator verbs status, offline, fmt and state-rollback
#   - report-mirror, the report-engine no-op and empty-trash-pipeline
#
# The secrets are the names in op.env. The toolbox gets their values from the vault in
# 1Password. It gives each value to the image with its name, not on a command line. Of
# these values, only the age identity goes into a file on the disk: mode 0600, for one
# call. Each step runs in the toolbox image, on a laptop and in CI.
version: '3'

vars:
  # The full Proton walk, weekly. The due verb of the toolbox counts the hours from the
  # last completed walk.
  RECONCILE_HOURS: 168

# This env: section is the environment of each migrator command in a run. RUN_BUDGET_MIN
# is a task var. A run sets it after the double dash (task sync -- RUN_BUDGET_MIN=30), or
# in the vars input of the workflow. This line gives it to the migrator with the same
# name. If the var is unset, its value is empty, and the migrator reads an empty value as
# unset. A run also sets RECONCILE after the double dash, but the due verb of the toolbox
# reads it, not the migrator.
#
# The work directory and the config path of the migrator are its defaults: .run and
# config/mirror.toml. AWS_REGION is an ENV line in the image. Thus, each command in the
# toolbox gets it.
env:
  RUN_BUDGET_MIN: '{{.RUN_BUDGET_MIN}}'
  # The log level of the Proton CLI. Its log lines go to the job log, which all persons
  # can read.
  PROTON_DRIVE_LOG_LEVEL: INFO

includes:
  toolbox:
    taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml
    flatten: true
    excludes: [report-mirror]
    vars:
      NAME: dropbox
      DESC: nightly Dropbox -> Proton Drive mirror; R2 holds the state
      PASS: MIRROR_VERBOSE
      MENU: >-
        ""
        "${c}read -- looks at Dropbox, Proton and R2, changes nothing${r}"
        "  task status               Counts and the last run's figures from the state in R2"
        ""
        "${c}checks -- offline, no credentials${r}"
        "  task run -- task offline  pytest, ruff check and ruff format --check"
        "  task fmt                  ruff format"
        ""
        "${c}write -- changes Proton Drive and the state in R2${r}"
        "  task sync -- RUN_BUDGET_MIN=30 RECONCILE=true"
        "                            ${d}a shorter budget; a forced Proton walk (the literal word true)${r}"
        "  task empty-trash          Permanently delete Proton trash (never scheduled)"
        "  task state-rollback -- k  Copy a history object over the canonical state"
        "                            ${d}with no key it lists the history${r}"
        "  task session-seal -- dir  Encrypt a laptop Proton CLI session into R2"
  # The proton engine supplies session, session-seal, empty-trash, and the pull and push
  # that the due and reconciled verbs of the toolbox use. This file contains a different
  # version of each task in the excludes: list.
  proton:
    taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/engines/proton.yml
    flatten: true
    excludes: [pipeline, plan-pipeline, trash, empty-trash-pipeline, report-engine]

tasks:
  # ---- pipeline (in the toolbox; each step of this file is one migrator command) ----
  pipeline:
    desc: The full budgeted run, inside the toolbox
    cmds:
      - task: clock
      - task: due
      - task: clock-phase
      - task: session
      - task: state
      - task: inventory
      - task: delta
      - task: plan-phase
      - task: batches
      - task: trash
      - task: reconcile
      - task: report-phase
      - task: report
      - task: ping

  plan-pipeline:
    desc: Read-only half of the pipeline, inside the toolbox; prints the report it would make
    cmds:
      - task: clock
      - task: due
      - task: clock-phase
      - task: session
      - task: state
      - task: inventory
      - task: delta
      - task: plan-phase
      # No --apply: the plan is read-only, and the report phase pushes nothing to R2.
      - python -m migrator report
      - cat {{.RUN}}/report.md

  # The clock step of the migrator runs after the clock of the toolbox. It reads the start
  # epoch from .run/start.txt and writes it, with its hour and weekday, to .run/clock.json.
  # It also removes the files in staging, the report and the walked marker.
  clock-phase: { internal: true, cmds: ['python -m migrator clock'] }
  state:      { internal: true, cmds: ['python -m migrator state'] }
  inventory:  { internal: true, cmds: ['python -m migrator inventory'] }
  delta:      { internal: true, cmds: ['python -m migrator delta'] }
  plan-phase: { internal: true, cmds: ['python -m migrator plan'] }
  batches:    { internal: true, cmds: ['python -m migrator --apply batches'] }
  trash:      { internal: true, cmds: ['python -m migrator --apply trash'] }
  reconcile:
    internal: true
    cmds:
      - python -m migrator --apply reconcile
      - task: record-walk
  # This task records a walk in R2 only if the walk completed. After a walk that did not
  # complete, the next run is due again, and it continues the walk.
  record-walk:
    internal: true
    status: ['test ! -f {{.RUN}}/walked']
    cmds: [{ task: reconciled }]
  # It writes .run/report.md from the state, completes the run row and writes the chain
  # marker.
  report-phase: { internal: true, cmds: ['python -m migrator --apply report'] }
  # The empty-trash verb of the engine uses `task op` to start this task in the image.
  # go-task does not start an internal task from the command line. Thus, this task is not
  # internal.
  empty-trash-pipeline:
    cmds:
      - task: clock
      - task: clock-phase
      - task: session
      - task: state
      - python -m migrator --apply empty-trash

  # The report verb of the toolbox runs this task, and then report-mirror. The pipeline of
  # this mirror does not use the stage and confirm verbs of the engine. Thus, this task
  # adds no rows of the engine.
  report-engine: {cmds: []}

  # The report verb of the toolbox runs this task after the rows of the toolbox. It runs at
  # the end of the pipeline, and on the failure path after a run that stopped at a step. If
  # a run stopped before its report phase, this task runs the report phase, which completes
  # the run row. Then the report goes on the job page, below the rows of the toolbox.
  report-mirror:
    internal: true
    cmds:
      - test -f {{.RUN}}/report.md || python -m migrator --apply report || true
      - test ! -f {{.RUN}}/report.md || cat {{.RUN}}/report.md >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"

  # ---- read ----
  status:
    desc: Counts from the state in R2
    cmds:
      - task: op
        vars: { ARGS: python -m migrator status }

  # ---- checks ----
  offline:
    desc: 'pytest, ruff check and ruff format --check, inside the toolbox: task run -- task offline'
    cmds:
      - python -m pytest
      - ruff check src tests
      - ruff format --check src tests
      # empty-trash starts empty-trash-pipeline with its name in the image. A dry run shows
      # that go-task accepts that name from the command line.
      - task --dry empty-trash-pipeline >/dev/null 2>&1 || { echo "empty-trash-pipeline is internal" >&2; exit 1; }

  fmt:
    desc: ruff format
    cmds:
      - task run -- ruff format src tests

  # ---- write ----
  state-rollback:
    desc: 'Copy a history object over the canonical state: task state-rollback -- <key>'
    cmds:
      - task: op
        vars: { ARGS: 'python -m migrator state-rollback {{.CLI_ARGS}}' }