Create a Magic ETL dataflow. Executes immediately after creation — no UI save required. TOP-LEVEL STRUCTURE: { "name": "...", "databaseType": "MAGIC", "draft": false, "enabled": true, "runState": "ENABLED", "engineProperties": { "kettle.mode": "STRICT" }, "computeCloud": "", // REQUIRED if inputs are Snowflake native — get from dataset metadata cloudId "inputs": [...], "outputs": [...], "actions": [...], "gui": { "version": "1.0", "useGraphUI": true, "canvases": { "default": { "canvasSettings": {}, "elements": [...], "disabledActions": [] } } } } INPUTS: [{ "dataSourceId": "", "dataSourceName": "...", "executeFlowWhenUpdated": false, "onlyLoadNewVersions": false, "recentVersionCutoffMs": 0 }] OUTPUTS: [{ "dataSourceId": null, "dataSourceName": "...", "versionChainType": "REPLACE" }] (null = auto-create on first run) ACTION TYPES (each has id, type, name, dependsOn[], settings: { preferredDatabaseEntityType: "TEMP_VIEW" }, gui, tables: [{}]): LoadFromVault (input): { "type": "LoadFromVault", "id": "LoadFromVault-xxx", "dataSourceId": "", "sourceType": "AUTO", "propagateAi": false, "filterPolicy": "LEGACY" } MergeJoin: { "type": "MergeJoin", "joinType": "LEFT OUTER|INNER|RIGHT OUTER|FULL OUTER", "relationshipType": "MTM" (NOT MANY_TO_MANY), "step1": "", "step2": "", "keys1": ["col"], "keys2": ["col"], "schemaModification2": [{ "name": "col", "rename": "", "remove": true }] } GroupBy: { "type": "GroupBy", "input": "", "groups": [{"name":"col"}], "fields": [{ "name": "Output Name", "source": "col", "type": "SUM|AVERAGE|COUNT_ALL|MIN|MAX", "valuefield": null, "expression": null, "settings": null }] } Expression aggregation: { "name": "...", "source": null, "type": null, "expression": "SUM(IFNULL(`col`,0))", ... } ExpressionEvaluator: { "type": "ExpressionEvaluator", "input": "", "expressions": [{ "expression": "YEAR(`date_col`)", "fieldName": "YEAR", "settings": {} }] } WindowAction: { "type": "WindowAction", "input": "", "groupRules": [{"column":"col","caseSensitive":false}], "orderRules": [{"column":"col","ascending":true}], "additions": [{"name":"Rank","operation":{"type":"RANKING","operationType":"ROW_NUMBER"}}] } SelectValues: { "type": "SelectValues", "input": "", "fields": [{ "name": "col", "rename": null, "remove": false }] } PublishToVault: { "type": "PublishToVault", "dependsOn": [""], "inputs": [""], "settings": { "preferredDatabaseEntityType": "DYNAMIC_TABLE" }, "dataSource": { "guid": null, "type": "DataFlow", "name": "..." }, "versionChainType": "REPLACE", "schemaSource": "DATAFLOW", "partitioned": false } CANVAS GUI: Organize tiles left-to-right in colored sections: Section: { "id": "", "type": "Section", "x": 24, "y": 24, "width": 384, "height": 560, "name": "Inputs", "backgroundColor": "var(--colorChartBlue6)" } Tile: { "id": "", "type": "Tile", "x": 72, "y": 56, "parentId": "" } Colors: Blue6=inputs, Orange6=transforms, Green6=aggregation, Purple6=output. SNOWFLAKE CLOUD: If input datasets have cloudEngine:"SNOWFLAKE", set computeCloud to their cloudId. Check dataset metadata first. Mismatched clouds cause indexing failures.