CLI: Deploy through v1 API with v2 spec

.claude/yaml-formats.md

@@ -0,0 +1,96 @@
+# OpenFn Project YAML Formats
+
+Two YAML formats are used across the monorepo. The key distinction: **v1** uses objects keyed by ID; **v2** uses arrays.
+
+## v1 (Lightning app state)
+
+Used by `packages/deploy` and sent to/from the Lightning API (`Provisioner.Project` type from `@openfn/lexicon/lightning`).
+
+- `workflows` is a keyed object (`{ [slug]: Workflow }`)
+- Each workflow has `jobs`, `triggers`, and `edges` as keyed objects
+- Steps are called `jobs`; code is stored in `body`
+- Credentials referenced by UUID (`project_credential_id`)
+- No version marker — absence of `schema_version`/`cli.version` means v1
+
+```yaml
+id: abc-123
+name: My Project
+project_credentials:
+  - id: cred-uuid
+    name: My Credential
+    owner: admin@openfn.org
+workflows:
+  my-workflow:
+    id: wf-uuid
+    name: My Workflow
+    jobs:
+      transform-data:
+        id: job-uuid
+        name: Transform data
+        body: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        project_credential_id: cred-uuid
+        keychain_credential_id: null
+    triggers:
+      webhook:
+        id: trig-uuid
+        type: webhook
+        enabled: true
+    edges:
+      trigger->transform-data:
+        id: edge-uuid
+        enabled: true
+        source_trigger_id: trig-uuid
+        target_job_id: job-uuid
+```
+
+## v2 (local project state)
+
+Used by `packages/project` and the CLI project subcommands (`ProjectState` type from `@openfn/lexicon`).
+
+- Identified by `schema_version` field (current: `'4.0'`) or legacy `cli.version: 2`
+- `workflows` is an array
+- Each workflow has a `steps` array; triggers are steps with a `type` field
+- Code stored in `expression`; edges expressed inline via `next` map on each step
+- Credentials referenced by name string (`configuration`)
+
+```yaml
+id: my-project
+name: My Project
+schema_version: '4.0'
+credentials:
+  - uuid: cred-uuid
+    name: My Credential
+    owner: admin@openfn.org
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data:
+            condition: always
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        configuration: 'admin@openfn.org|My Credential'
+```
+
+## Detection logic
+
+Use `detectVersion(data)` from `@openfn/project` — returns `1` or `2`. Accepts YAML/JSON string or pre-parsed object.
+
+```typescript
+import { detectVersion } from '@openfn/project';
+if (detectVersion(json) === 2) { /* v2 */ }
+```
+
+## Conversion
+
+- **v2 → v1**: `Project.from('project', json).then(p => p.serialize('state', { format: 'yaml' }))` — see `maybeConvertV2spec` in `packages/cli/src/deploy/handler.ts`
+- **v1 → v2**: `Project.from('state', json)` — see `packages/project/src/parse/from-app-state.ts`
+- Full conversion logic: `packages/project/src/serialize/to-app-state.ts` (v2→v1) and `packages/project/src/parse/from-app-state.ts` (v1→v2)

CLAUDE.md

@@ -131,6 +131,7 @@ cd packages/cli && pnpm test:watch  # Watch mode
 The [.claude](.claude) folder contains detailed guides:
 
 - **[event-processor.md](.claude/event-processor.md)** - Worker event processing deep-dive (ordering, batching) — companion to `packages/ws-worker/CLAUDE.md`
+- **[yaml-formats.md](.claude/yaml-formats.md)** - v1 vs v2 project YAML formats: structure, detection logic, and conversion paths
 
 Key packages also carry their own `CLAUDE.md` (runtime, engine-multi, ws-worker), auto-loaded when you work in them.
 

integration-tests/cli/test/deploy.test.ts

@@ -269,6 +269,52 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
   );
 });
 
+test.serial('deploy a v2 spec file', async (t) => {
+  const testProjectV2 = `
+name: test-project
+schema_version: '4.0'
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          my-job: {}
+      - id: my-job
+        name: My Job
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+`.trim();
+
+  await fs.writeFile(path.join(tmpDir, 'project.yaml'), testProjectV2);
+
+  t.is(Object.keys(server.state.projects).length, 0);
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --state-path ${tmpDir}/.state.json \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /v2 spec/i);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'test-project');
+  const [workflow] = Object.values(project.workflows) as any[];
+  t.is(workflow.name, 'My Workflow');
+});
+
 test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   t.is(Object.keys(server.state.projects).length, 0);
 

packages/cli/src/deploy/handler.ts

@@ -10,7 +10,7 @@ import { DeployOptions } from './command';
 import * as beta from '../projects/deploy';
 import path from 'node:path';
 import { fileExists } from '../util/file-exists';
-import { yamlToJson } from '@openfn/project';
+import Project, { detectVersion, yamlToJson } from '@openfn/project';
 import fs from 'node:fs/promises';
 
 export type DeployFn = typeof deploy;
@@ -62,6 +62,15 @@ async function deployHandler(
       config.endpoint = process.env['OPENFN_ENDPOINT'];
     }
 
+    const rawSpec = await fs.readFile(config.specPath, 'utf-8');
+    const convertedSpec = await maybeConvertV2spec(rawSpec);
+    if (convertedSpec !== rawSpec) {
+      logger.info(
+        'Detected v2 spec file - converting to legacy format; validation will be skipped.'
+      );
+      config.spec = convertedSpec;
+    }
+
     logger.debug('Deploying with config', config);
     logger.info(`Deploying`);
 
@@ -137,4 +146,13 @@ const redirectTov2 = async (
   );
 };
 
+export const maybeConvertV2spec = async (yaml: string): Promise<string> => {
+  const json = yamlToJson(yaml) as any;
+  if (detectVersion(json) > 1) {
+    const project = await Project.from('project', json);
+    return project.serialize('state', { format: 'yaml' }) as string;
+  }
+  return yaml;
+};
+
 export default deployHandler;

packages/cli/test/deploy/deploy.test.ts

@@ -3,7 +3,11 @@
 import test from 'ava';
 import mockfs from 'mock-fs';
 import { Logger, createMockLogger } from '@openfn/logger';
-import deployHandler, { DeployFn } from '../../src/deploy/handler';
+import deployHandler, {
+  DeployFn,
+  maybeConvertV2spec,
+} from '../../src/deploy/handler';
+import { yamlToJson } from '@openfn/project';
 
 import { DeployError, type DeployConfig } from '@openfn/deploy';
 import { DeployOptions } from '../../src/deploy/command';
@@ -183,3 +187,96 @@ test.serial('catches DeployErrors', async (t) => {
   t.is(process.exitCode, 10);
   process.exitCode = origExitCode;
 });
+
+// maybeConvertV2spec
+
+const v1Yaml = `id: '1234'
+name: My Project
+workflows:
+  my-workflow:
+    id: job-1
+    name: My Workflow
+    jobs:
+      transform-data:
+        id: job-1
+        name: Transform data
+        body: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        project_credential_id: null
+        keychain_credential_id: null
+    triggers:
+      webhook:
+        id: trig-1
+        type: webhook
+        enabled: true
+    edges:
+      trigger->transform-data:
+        id: edge-1
+        enabled: true
+        source_trigger_id: trig-1
+        target_job_id: job-1
+project_credentials: []
+`;
+
+const v2Yaml = `id: my-project
+name: My Project
+schema_version: '4.0'
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data: {}
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+`;
+
+test('maybeConvertV2spec: returns v1 yaml unchanged', async (t) => {
+  const result = await maybeConvertV2spec(v1Yaml);
+  t.is(result, v1Yaml);
+});
+
+test('maybeConvertV2spec: converts v2 (schema_version) to v1', async (t) => {
+  const result = await maybeConvertV2spec(v2Yaml);
+  const json = yamlToJson(result) as any;
+
+  // v1 has workflows as a keyed object
+  t.is(typeof json.workflows, 'object');
+  t.false(Array.isArray(json.workflows));
+
+  // v1 uses jobs, not steps
+  const workflow = Object.values(json.workflows)[0] as any;
+  t.truthy(workflow.jobs);
+  t.falsy(workflow.steps);
+  t.truthy(workflow.triggers);
+
+  // no v2 marker
+  t.falsy(json.schema_version);
+});
+
+test('maybeConvertV2spec: converts legacy v2 (cli.version: 2) to v1', async (t) => {
+  const legacyV2Yaml = `id: my-project
+name: My Project
+cli:
+  version: 2
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+`;
+  const result = await maybeConvertV2spec(legacyV2Yaml);
+  const json = yamlToJson(result) as any;
+
+  t.is(typeof json.workflows, 'object');
+  t.false(Array.isArray(json.workflows));
+});

packages/deploy/src/index.ts

@@ -2,7 +2,7 @@ import { confirm } from '@inquirer/prompts';
 import { inspect } from 'node:util';
 import { DeployConfig, ProjectState } from './types';
 import { readFile, writeFile } from 'fs/promises';
-import { parseAndValidate } from './validator';
+import { parseAndValidate, parseSpec } from './validator';
 import jsondiff from 'json-diff';
 import {
   mergeProjectPayloadIntoState,
@@ -108,8 +108,8 @@ export async function getSpec(path: string) {
 
 export async function deploy(config: DeployConfig, logger: Logger) {
   const [state, spec] = await Promise.all([
-    getState(config.statePath),
-    getSpec(config.specPath),
+    config.state ?? getState(config.statePath),
+    config.spec ? parseSpec(config.spec) : getSpec(config.specPath),
   ]);
 
   logger.debug('spec', spec);

packages/deploy/src/types.ts

@@ -193,4 +193,6 @@ export interface DeployConfig {
   requireConfirmation: boolean;
   dryRun: boolean;
   apiKey: string | null;
+  spec?: string;
+  state?: ProjectState;
 }

packages/deploy/src/validator.ts

@@ -3,6 +3,10 @@ import { ProjectSpec } from './types';
 import { readFile } from 'fs/promises';
 import path from 'path';
 
+export function parseSpec(input: string) {
+  return { errors: [] as Error[], doc: YAML.parse(input) as ProjectSpec };
+}
+
 export interface Error {
   context: any;
   message: string;

packages/project/src/index.ts

@@ -24,4 +24,6 @@ export {
 
 export { mapWorkflow } from './parse/from-app-state';
 
+export { default as detectVersion } from './util/detect-version';
+
 export type { MergeProjectOptions } from './merge/merge-project';

packages/project/src/parse/from-project.ts

@@ -4,6 +4,7 @@ import Project from '../Project';
 import ensureJson from '../util/ensure-json';
 import { Provisioner } from '@openfn/lexicon/lightning';
 import fromAppState, { fromAppStateConfig } from './from-app-state';
+import detectVersion from '../util/detect-version';
 
 // Load a project from any JSON or yaml representation
 // This is backwards-compatible with v1 state.json files
@@ -21,11 +22,7 @@ export default (
   // first ensure the data is in JSON format
   let rawJson = ensureJson<any>(data);
 
-  if (
-    rawJson.schema_version ||
-    rawJson.cli?.version === 2 ||
-    rawJson.version /*deprecated*/
-  ) {
+  if (detectVersion(rawJson) > 1) {
     return new Project(from_v2(rawJson as SerializedProject), config);
   }
 

packages/project/src/util/detect-version.ts

@@ -0,0 +1,14 @@
+import ensureJson from './ensure-json';
+
+// Detect whether a project spec is v1 (Lightning app state) or v2 (local project state)
+// Accepts YAML/JSON strings or a pre-parsed object
+export default function detectVersion(projectSpec: string | object): number {
+  const json = ensureJson<any>(projectSpec);
+  if (json.schema_version) {
+    return parseInt(json.schema_version, 10);
+  }
+  if (json.cli?.version === 2 || json.version) {
+    return 2;
+  }
+  return 1;
+}