From 1de47627893a845aaa81b59e7ddce3a7de8069dc Mon Sep 17 00:00:00 2001 From: mattcasters Date: Tue, 10 Nov 2026 10:26:53 +0100 Subject: [PATCH] Issue #2838 : Add experimental Salesforce Bulk API 2.0 transforms The SOAP transforms stay synchronous. Bulk load and bulk input use the job API, and the integration test talks to a local stub instead of a Salesforce org. --- .../integration-tests-salesforce.yaml | 42 ++ .../salesforce-mock/bulk_api.py | 229 +++++++++++ docs/hop-user-manual/modules/ROOT/nav.adoc | 2 + .../transforms/salesforcebulkinput.adoc | 57 +++ .../transforms/salesforcebulkload.adoc | 70 ++++ .../pipeline/transforms/salesforceinput.adoc | 1 + .../pipeline/transforms/salesforceinsert.adoc | 1 + .../salesforce/0001-bulk-insert.hpl | 200 ++++++++++ .../salesforce/0002-bulk-insert-error.hpl | 234 +++++++++++ .../salesforce/0003-bulk-query.hpl | 161 ++++++++ .../salesforce/0004-bulk-update.hpl | 211 ++++++++++ .../salesforce/0005-bulk-delete.hpl | 199 ++++++++++ .../salesforce/0006-bulk-upsert.hpl | 211 ++++++++++ .../salesforce/dev-env-config.json | 11 + integration-tests/salesforce/hop-config.json | 290 ++++++++++++++ .../salesforce/main-0001-salesforce-bulk.hwf | 194 ++++++++++ .../pipeline-run-configuration/local.json | 17 + .../salesforce-mock.json | 8 + .../workflow-run-configuration/local.json | 9 + .../salesforce/project-config.json | 15 + .../samples/read-samples-build-hop-run.hpl | 8 + ...opPipelineMetaToBeamPipelineConverter.java | 6 +- .../org/apache/hop/beam/util/BeamConst.java | 8 + .../BeamPipelineEngineSupportsTest.java | 17 + plugins/engines/spark/pom.xml | 6 + .../HopPipelineMetaToSparkConverter.java | 6 +- .../org/apache/hop/spark/util/SparkConst.java | 2 + .../HopPipelineMetaToSparkConverterTest.java | 8 +- .../salesforcebulk/SalesforceBulkClient.java | 47 +++ .../SalesforceBulkConcurrency.java | 45 +++ .../salesforcebulk/SalesforceBulkCsv.java | 177 +++++++++ .../SalesforceBulkDelimiter.java | 55 +++ .../salesforcebulk/SalesforceBulkField.java | 54 +++ .../SalesforceBulkHttpClient.java | 227 +++++++++++ .../SalesforceBulkIngestRequest.java | 46 +++ .../salesforcebulk/SalesforceBulkInput.java | 223 +++++++++++ .../SalesforceBulkInputData.java | 45 +++ .../SalesforceBulkInputDialog.java | 74 ++++ .../SalesforceBulkInputField.java | 55 +++ .../SalesforceBulkInputMeta.java | 262 +++++++++++++ .../salesforcebulk/SalesforceBulkJob.java | 47 +++ .../SalesforceBulkLineEnding.java | 51 +++ .../salesforcebulk/SalesforceBulkLoad.java | 318 +++++++++++++++ .../SalesforceBulkLoadData.java | 48 +++ .../SalesforceBulkLoadDialog.java | 74 ++++ .../SalesforceBulkLoadMeta.java | 364 ++++++++++++++++++ .../SalesforceBulkOperation.java | 71 ++++ .../SalesforceBulkQueryOperation.java | 45 +++ .../SalesforceBulkQueryPage.java | 36 ++ .../SalesforceBulkRowMatcher.java | 120 ++++++ .../salesforcebulk/SalesforceBulkRuntime.java | 122 ++++++ .../salesforcebulk/SalesforceBulkSession.java | 97 +++++ .../salesforcebulk/SalesforceBulkValues.java | 95 +++++ .../messages/messages_en_US.properties | 129 +++++++ .../main/resources/salesforcebulkinput.svg | 28 ++ .../src/main/resources/salesforcebulkload.svg | 28 ++ .../transforms/salesforce-bulk-input.hpl | 99 +++++ .../transforms/salesforce-bulk-load.hpl | 142 +++++++ .../salesforcebulk/SalesforceBulkCsvTest.java | 66 ++++ .../SalesforceBulkHttpClientTest.java | 149 +++++++ .../SalesforceBulkLoadTest.java | 175 +++++++++ .../SalesforceBulkRowMatcherTest.java | 67 ++++ .../SalesforceBulkSessionTest.java | 67 ++++ .../SalesforceBulkValuesTest.java | 68 ++++ 64 files changed, 6036 insertions(+), 3 deletions(-) create mode 100644 docker/integration-tests/integration-tests-salesforce.yaml create mode 100644 docker/integration-tests/salesforce-mock/bulk_api.py create mode 100644 docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkinput.adoc create mode 100644 docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkload.adoc create mode 100644 integration-tests/salesforce/0001-bulk-insert.hpl create mode 100644 integration-tests/salesforce/0002-bulk-insert-error.hpl create mode 100644 integration-tests/salesforce/0003-bulk-query.hpl create mode 100644 integration-tests/salesforce/0004-bulk-update.hpl create mode 100644 integration-tests/salesforce/0005-bulk-delete.hpl create mode 100644 integration-tests/salesforce/0006-bulk-upsert.hpl create mode 100644 integration-tests/salesforce/dev-env-config.json create mode 100644 integration-tests/salesforce/hop-config.json create mode 100644 integration-tests/salesforce/main-0001-salesforce-bulk.hwf create mode 100644 integration-tests/salesforce/metadata/pipeline-run-configuration/local.json create mode 100644 integration-tests/salesforce/metadata/salesforce-connection/salesforce-mock.json create mode 100644 integration-tests/salesforce/metadata/workflow-run-configuration/local.json create mode 100644 integration-tests/salesforce/project-config.json create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkClient.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkConcurrency.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsv.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkDelimiter.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkField.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClient.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkIngestRequest.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInput.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputData.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputDialog.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputField.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputMeta.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkJob.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLineEnding.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoad.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadData.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadDialog.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadMeta.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkOperation.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryOperation.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryPage.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcher.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRuntime.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSession.java create mode 100644 plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValues.java create mode 100644 plugins/tech/salesforce/src/main/resources/org/apache/hop/pipeline/transforms/salesforcebulk/messages/messages_en_US.properties create mode 100644 plugins/tech/salesforce/src/main/resources/salesforcebulkinput.svg create mode 100644 plugins/tech/salesforce/src/main/resources/salesforcebulkload.svg create mode 100644 plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-input.hpl create mode 100644 plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-load.hpl create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsvTest.java create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClientTest.java create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadTest.java create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcherTest.java create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSessionTest.java create mode 100644 plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValuesTest.java diff --git a/docker/integration-tests/integration-tests-salesforce.yaml b/docker/integration-tests/integration-tests-salesforce.yaml new file mode 100644 index 00000000000..eb3a0f91e36 --- /dev/null +++ b/docker/integration-tests/integration-tests-salesforce.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + integration_test_salesforce: + extends: + file: integration-tests-base.yaml + service: integration_test + depends_on: + salesforce_mock: + condition: service_healthy + links: + - salesforce_mock + + salesforce_mock: + image: python:3.12-alpine + hostname: salesforce_mock + command: ["python", "/opt/bulk_api.py"] + volumes: + - ./salesforce-mock/bulk_api.py:/opt/bulk_api.py:ro + ports: + - "8080" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 2s diff --git a/docker/integration-tests/salesforce-mock/bulk_api.py b/docker/integration-tests/salesforce-mock/bulk_api.py new file mode 100644 index 00000000000..7cf3aa0061a --- /dev/null +++ b/docker/integration-tests/salesforce-mock/bulk_api.py @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A small Bulk API 2.0 stub for the Hop integration tests. + +It stores the uploaded CSV and returns it as success and failed results. A row +is a failure when any cell is the text FAIL. Result rows are reversed so the +Hop transform has to match them instead of trusting file order. +""" + +import csv +import io +import json +import re +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +TOKEN = "Bearer test-token" +JOBS = {} +COUNTER = 0 +DELIMITERS = { + "COMMA": ",", + "SEMICOLON": ";", + "TAB": "\t", + "PIPE": "|", + "CARET": "^", + "BACKQUOTE": "`", +} +INGEST = re.compile(r"^/services/data/v[^/]+/jobs/ingest(?:/([^/]+))?(?:/(batches|successfulResults|failedResults))?$") +QUERY = re.compile(r"^/services/data/v[^/]+/jobs/query(?:/([^/]+))?(?:/(results))?$") + + +def new_id(): + global COUNTER + COUNTER += 1 + return f"001{COUNTER:015d}" + + +def read_csv(text, delimiter): + reader = csv.reader(io.StringIO(text), delimiter=delimiter) + rows = [row for row in reader if any(cell != "" for cell in row)] + if not rows: + return [], [] + width = len(rows[0]) + normalized = [] + for row in rows: + padded = list(row) + [""] * (width - len(row)) + normalized.append(padded[:width]) + return normalized[0], normalized[1:] + + +def write_csv(header, rows, delimiter, line_ending): + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, delimiter=delimiter, lineterminator=line_ending, quoting=csv.QUOTE_MINIMAL) + writer.writerow(header) + for row in rows: + writer.writerow(row) + return buffer.getvalue() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + return + + def do_GET(self): + path = self.path.split("?", 1)[0] + if path == "/health": + self.respond(200, "ok", "text/plain") + return + if not self.authorized(): + return + ingest = INGEST.match(path) + if ingest: + self.get_ingest(ingest.group(1), ingest.group(2)) + return + query = QUERY.match(path) + if query: + self.get_query(query.group(1), query.group(2)) + return + self.respond(404, "not found", "text/plain") + + def do_POST(self): + if not self.authorized(): + return + body = self.body() + path = self.path.split("?", 1)[0] + if INGEST.match(path): + payload = json.loads(body.decode("utf-8")) + job_id = new_id() + JOBS[job_id] = { + "kind": "ingest", + "state": "Open", + "object": payload.get("object"), + "operation": payload.get("operation"), + "delimiter": DELIMITERS.get(payload.get("columnDelimiter"), ","), + "line_ending": "\r\n" if payload.get("lineEnding") == "CRLF" else "\n", + "csv": "", + } + self.respond(200, json.dumps({"id": job_id, "state": "Open"}), "application/json") + return + if QUERY.match(path): + job_id = new_id() + JOBS[job_id] = {"kind": "query", "state": "JobComplete"} + self.respond(200, json.dumps({"id": job_id, "state": "JobComplete"}), "application/json") + return + self.respond(404, "not found", "text/plain") + + def do_PUT(self): + if not self.authorized(): + return + ingest = INGEST.match(self.path.split("?", 1)[0]) + if not ingest or ingest.group(2) != "batches": + self.respond(404, "not found", "text/plain") + return + job = JOBS.get(ingest.group(1)) + if job is None: + self.respond(404, "not found", "text/plain") + return + job["csv"] = self.body().decode("utf-8") + self.respond(201, "", "text/plain") + + def do_PATCH(self): + if not self.authorized(): + return + path = self.path.split("?", 1)[0] + ingest = INGEST.match(path) + query = QUERY.match(path) + job_id = ingest.group(1) if ingest else query.group(1) if query else None + job = JOBS.get(job_id) if job_id else None + if job is None: + self.respond(404, "not found", "text/plain") + return + payload = json.loads(self.body().decode("utf-8") or "{}") + state = payload.get("state", "JobComplete") + job["state"] = "JobComplete" if state == "UploadComplete" else state + self.respond(200, json.dumps(self.job_json(job_id, job)), "application/json") + + def get_ingest(self, job_id, suffix): + job = JOBS.get(job_id) + if job is None: + self.respond(404, "not found", "text/plain") + return + if suffix is None: + self.respond(200, json.dumps(self.job_json(job_id, job)), "application/json") + return + header, rows = read_csv(job.get("csv", ""), job["delimiter"]) + success = [] + failed = [] + for row in reversed(rows): + if any(cell == "FAIL" for cell in row): + failed.append(row + ["", "simulated failure"]) + else: + id_index = header.index("Id") if "Id" in header else -1 + sf_id = row[id_index] if id_index >= 0 and row[id_index] else new_id() + success.append(row + [sf_id, "true"]) + if suffix == "successfulResults": + text = write_csv(header + ["sf__Id", "sf__Created"], success, job["delimiter"], job["line_ending"]) + else: + text = write_csv(header + ["sf__Id", "sf__Error"], failed, job["delimiter"], job["line_ending"]) + self.respond(200, text, "text/csv") + + def get_query(self, job_id, suffix): + job = JOBS.get(job_id) + if job is None: + self.respond(404, "not found", "text/plain") + return + if suffix is None: + self.respond(200, json.dumps(self.job_json(job_id, job)), "application/json") + return + locator = self.headers.get("Sforce-Locator") + if locator == "page2": + body = "Id,Name\n001000000000003AAA,Beta\n" + next_locator = "null" + else: + body = "Id,Name\n001000000000001AAA,Acme\n001000000000002AAA,\n" + next_locator = "page2" + self.send_response(200) + encoded = body.encode("utf-8") + self.send_header("Content-Type", "text/csv; charset=utf-8") + self.send_header("Content-Length", str(len(encoded))) + self.send_header("Sforce-Locator", next_locator) + self.end_headers() + self.wfile.write(encoded) + + def job_json(self, job_id, job): + return { + "id": job_id, + "state": job["state"], + "operation": job.get("operation"), + "object": job.get("object"), + "numberRecordsProcessed": 0, + "numberRecordsFailed": 0, + } + + def authorized(self): + if self.headers.get("Authorization") == TOKEN: + return True + self.respond(401, '[{"errorCode":"INVALID_SESSION_ID"}]', "application/json") + return False + + def body(self): + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) if length else b"" + + def respond(self, status, text, content_type): + encoded = text.encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + +if __name__ == "__main__": + server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler) + server.serve_forever() diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 95ff2ed48d5..8deaa631e37 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -297,6 +297,8 @@ under the License. *** xref:pipeline/transforms/rowgenerator.adoc[Row Generator] *** xref:pipeline/transforms/rownormaliser.adoc[Row Normaliser] *** xref:pipeline/transforms/runssh.adoc[Run SSH commands] +*** xref:pipeline/transforms/salesforcebulkinput.adoc[Salesforce bulk input] +*** xref:pipeline/transforms/salesforcebulkload.adoc[Salesforce bulk load] *** xref:pipeline/transforms/salesforcedelete.adoc[Salesforce Delete] *** xref:pipeline/transforms/salesforceinput.adoc[Salesforce Input] *** xref:pipeline/transforms/salesforceinsert.adoc[Salesforce Insert] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkinput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkinput.adoc new file mode 100644 index 00000000000..740872220de --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkinput.adoc @@ -0,0 +1,57 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US + +:description: Experimental transform that reads a large SOQL query with Bulk API 2.0. +:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=no, Beam Spark=no, Beam Flink=no, Beam Dataflow=no +:page-marketplace: hop-tech-salesforce + += image:transforms/icons/salesforcebulkinput.svg[Salesforce bulk input transform icon, role="image-doc-icon"] Salesforce bulk input + +== Description + +WARNING: This transform is experimental. The dialog and the saved pipeline may still change. + +Salesforce bulk input runs a SOQL query as a Bulk API 2.0 query job and reads the CSV result pages. +Use it for a large extract. +xref:pipeline/transforms/salesforceinput.adoc[Salesforce Input] remains the SOAP query, including module browse and the deleted-record filter. + +`Query all` includes deleted and archived rows. +Bulk query does not support aggregates, `GROUP BY`, `OFFSET`, or `TYPEOF`. +Primary-key chunking is automatic. +An empty CSV field becomes a null. +Dates are read as UTC timestamps in ISO-8601 form. + +The connection is the same xref:metadata-types/salesforce-connection.adoc[Salesforce connection] used by Salesforce bulk load. + +== Options + +* *Salesforce connection*. The connection metadata. +* *Operation*. Query, or query all. +* *SOQL*. The query text. Variables are resolved. +* *Timeout* and *Poll interval*. Seconds to wait for the job, and how often to ask for its state. +* *Maximum rows*. Stop after this many rows. 0 reads every row Salesforce returns. +* *Job id field*. Optional output field for the Bulk API job id. + +Each output field name must match a column in the result, including case. +The type is String, Integer, Number, BigNumber, Date, Timestamp, or Boolean. + +== Limits + +A query job can return up to 15 files of 1 GB. +An org can run 10,000 query jobs and return 1 TB of query results per rolling 24 hours. diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkload.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkload.adoc new file mode 100644 index 00000000000..13e58a70fb3 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforcebulkload.adoc @@ -0,0 +1,70 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US + +:description: Experimental transform that loads Salesforce records with Bulk API 2.0. +:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=no, Beam Spark=no, Beam Flink=no, Beam Dataflow=no +:page-marketplace: hop-tech-salesforce + += image:transforms/icons/salesforcebulkload.svg[Salesforce bulk load transform icon, role="image-doc-icon"] Salesforce bulk load + +== Description + +WARNING: This transform is experimental. The dialog and the saved pipeline may still change. + +Salesforce bulk load inserts, updates, upserts, deletes, or hard-deletes records through Bulk API 2.0. +It collects the input rows into a CSV file, creates one ingest job, and waits until Salesforce finishes it. +A job that grows past *Maximum bytes per job* is closed and another job is started. +The five SOAP transforms (Input, Insert, Update, Upsert, and Delete) are unchanged and remain the right choice for a small synchronous call. + +The transform reuses a xref:metadata-types/salesforce-connection.adoc[Salesforce connection]. +An OAuth or JWT connection supplies the access token. +A username and password connection logs in with the SOAP API and sends that session id as the bearer token. + +Bulk API 2.0 speaks CSV only and handles one Salesforce object per job. +On update and upsert, a null value is written as `#N/A` so Salesforce clears the field. +An empty string does not clear an existing value. +On insert, a null is an empty field. +Result files are not in input order. +Rows are matched on the id for update and delete, on the external id for upsert, and on every mapped value for insert. +Two identical input rows are paired in input order, which is ambiguous if Salesforce also rewrites the values. + +A failed row is sent to the error hop when one is connected. +Otherwise the row is written to the main output with the success field set to false. +Rows that Salesforce already committed are not rolled back. + +== Options + +* *Salesforce connection*. The connection metadata. +* *Salesforce object*. The object API name, for example `Account`. +* *Operation*. Insert, update, upsert, delete, or hard delete. +* *External id field*. Required for upsert. The same field must be in the field map. +* *Concurrency*. Parallel is faster. Serial avoids row-lock errors when many rows share a parent. +* *Line ending* and *Column delimiter*. Written into the CSV and declared on the job. +* *Timeout* and *Poll interval*. Seconds to wait for a job, and how often to ask for its state. The defaults are 3600 and 10. +* *Maximum bytes per job*. Default 104857600, which is under the 150 MB Bulk API limit. +* *Salesforce id field*, *Job id field*, *Success field*, and *Error field*. Output fields. Leave a name empty to omit that field. + +The field map names a stream field and the Salesforce field API name. +The CSV header uses that API name, including its case. + +== Limits + +Bulk API 2.0 has no service level agreement. +A job can stay open for 24 hours, and Salesforce keeps the result files for 7 days. +Daily org limits include 150 million records and 15,000 batches, shared with Bulk API 1.0. diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinput.adoc index 53d718f34ef..dab03ba8d1a 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinput.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinput.adoc @@ -32,6 +32,7 @@ You can also use the following transforms for various ways to modify your Salesf * xref:pipeline/transforms/salesforceupdate.adoc[Salesforce Update]: Updates records directly into your Salesforce database. * xref:pipeline/transforms/salesforceupsert.adoc[Salesforce Upsert]: Updates existing records and inserts new records into your Salesforce database. * xref:pipeline/transforms/salesforcedelete.adoc[Salesforce Delete]: Deletes records directly from your Salesforce database. +* xref:pipeline/transforms/salesforcebulkinput.adoc[Salesforce bulk input]: Experimental Bulk API 2.0 query for a large extract. == General diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinsert.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinsert.adoc index 31b4a69be0d..eefc64d2496 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinsert.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/salesforceinsert.adoc @@ -32,6 +32,7 @@ You can also use the following other transforms to modify your Salesforce databa * xref:pipeline/transforms/salesforceupdate.adoc[Salesforce Update]: Updates records directly into your Salesforce database. * xref:pipeline/transforms/salesforceupsert.adoc[Salesforce Upsert]: Updates existing records and inserts new records into your Salesforce database. * xref:pipeline/transforms/salesforcedelete.adoc[Salesforce Delete]: Deletes records directly from your Salesforce database. +* xref:pipeline/transforms/salesforcebulkload.adoc[Salesforce bulk load]: Experimental Bulk API 2.0 load for a large insert, update, upsert, or delete. == General diff --git a/integration-tests/salesforce/0001-bulk-insert.hpl b/integration-tests/salesforce/0001-bulk-insert.hpl new file mode 100644 index 00000000000..5193169241f --- /dev/null +++ b/integration-tests/salesforce/0001-bulk-insert.hpl @@ -0,0 +1,200 @@ + + + + + 0001-bulk-insert + Y + Normal + + + + rows + bulk + Y + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + Name + String + + -1 + -1 + N + + + + Alice + Bob + + + + 80 + 80 + + + + bulk + SalesforceBulkLoad + + Y + + 1 + + none + + + salesforce-mock + Account + insert + + Parallel + LF + COMMA + 60 + 1 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + Name + Name + + + + + 240 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + Name + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 2 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + + diff --git a/integration-tests/salesforce/0002-bulk-insert-error.hpl b/integration-tests/salesforce/0002-bulk-insert-error.hpl new file mode 100644 index 00000000000..c88609069f5 --- /dev/null +++ b/integration-tests/salesforce/0002-bulk-insert-error.hpl @@ -0,0 +1,234 @@ + + + + + 0002-bulk-insert-error + Y + Normal + + + + rows + bulk + Y + + + bulk + accepted + Y + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + Name + String + + -1 + -1 + N + + + + FAIL + + + + 80 + 80 + + + + bulk + SalesforceBulkLoad + + Y + + 1 + + none + + + salesforce-mock + Account + insert + + Parallel + LF + COMMA + 60 + 1 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + Name + Name + + + + + 240 + 80 + + + + accepted + Dummy + + Y + + 1 + + none + + + + + + 400 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + SalesforceError + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 1 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + bulk + Count rows + Y + + SalesforceError + + + + + + + + + diff --git a/integration-tests/salesforce/0003-bulk-query.hpl b/integration-tests/salesforce/0003-bulk-query.hpl new file mode 100644 index 00000000000..9a3983abf13 --- /dev/null +++ b/integration-tests/salesforce/0003-bulk-query.hpl @@ -0,0 +1,161 @@ + + + + + 0003-bulk-query + Y + Normal + + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + bulk + SalesforceBulkInput + + Y + + 1 + + none + + + salesforce-mock + query + SELECT Id, Name FROM Account + 60 + 1 + 0 + + + + Id + String + + + Name + String + + + + + 80 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + Id + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 3 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + + diff --git a/integration-tests/salesforce/0004-bulk-update.hpl b/integration-tests/salesforce/0004-bulk-update.hpl new file mode 100644 index 00000000000..d20cd7c0cf2 --- /dev/null +++ b/integration-tests/salesforce/0004-bulk-update.hpl @@ -0,0 +1,211 @@ + + + + + 0004-bulk-update + Y + Normal + + + + rows + bulk + Y + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + Id + String + + -1 + -1 + N + + + Name + String + + -1 + -1 + N + + + + 001000000000009AAAAda + + + + 80 + 80 + + + + bulk + SalesforceBulkLoad + + Y + + 1 + + none + + + salesforce-mock + Account + update + + Parallel + LF + COMMA + 60 + 1 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + Id + Id + + + Name + Name + + + + + 240 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + Id + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 1 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + + diff --git a/integration-tests/salesforce/0005-bulk-delete.hpl b/integration-tests/salesforce/0005-bulk-delete.hpl new file mode 100644 index 00000000000..f063f397b61 --- /dev/null +++ b/integration-tests/salesforce/0005-bulk-delete.hpl @@ -0,0 +1,199 @@ + + + + + 0005-bulk-delete + Y + Normal + + + + rows + bulk + Y + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + Id + String + + -1 + -1 + N + + + + 001000000000009AAA + + + + 80 + 80 + + + + bulk + SalesforceBulkLoad + + Y + + 1 + + none + + + salesforce-mock + Account + delete + + Parallel + LF + COMMA + 60 + 1 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + Id + Id + + + + + 240 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + Id + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 1 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + + diff --git a/integration-tests/salesforce/0006-bulk-upsert.hpl b/integration-tests/salesforce/0006-bulk-upsert.hpl new file mode 100644 index 00000000000..c2845869932 --- /dev/null +++ b/integration-tests/salesforce/0006-bulk-upsert.hpl @@ -0,0 +1,211 @@ + + + + + 0006-bulk-upsert + Y + Normal + + + + rows + bulk + Y + + + bulk + Count rows + Y + + + Count rows + Wrong count? + Y + + + Wrong count? + Abort + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + ExtId + String + + -1 + -1 + N + + + Name + String + + -1 + -1 + N + + + + E1Ada + + + + 80 + 80 + + + + bulk + SalesforceBulkLoad + + Y + + 1 + + none + + + salesforce-mock + Account + upsert + ExtId__c + Parallel + LF + COMMA + 60 + 1 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + ExtId + ExtId__c + + + Name + Name + + + + + 240 + 80 + + + + Count rows + MemoryGroupBy + + Y + + 1 + + none + + + Y + + + + rows_read + ExtId + COUNT_ANY + + + + + + 400 + 80 + + + + Wrong count? + FilterRows + + Y + + 1 + + none + + + + + N + rows_read + <> + + + constant + Integer + 1 + -1 + 0 + N + ####0;-####0 + + + + + + 560 + 80 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 720 + 80 + + + + + + diff --git a/integration-tests/salesforce/dev-env-config.json b/integration-tests/salesforce/dev-env-config.json new file mode 100644 index 00000000000..4d5bdfdc2d3 --- /dev/null +++ b/integration-tests/salesforce/dev-env-config.json @@ -0,0 +1,11 @@ +{ + "variables" : [ { + "name" : "sample-var2", + "value" : "ProvaDev2", + "description" : "" + }, { + "name" : "sample-var1", + "value" : "ProvaDev1", + "description" : "" + } ] +} \ No newline at end of file diff --git a/integration-tests/salesforce/hop-config.json b/integration-tests/salesforce/hop-config.json new file mode 100644 index 00000000000..d9e1e6562e0 --- /dev/null +++ b/integration-tests/salesforce/hop-config.json @@ -0,0 +1,290 @@ +{ + "variables": [ + { + "name": "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION", + "value": "N", + "description": "System wide flag to allow lenient string to number conversion for backward compatibility. If this setting is set to \"Y\", an string starting with digits will be converted successfully into a number. (example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending on the decimal and grouping symbol). The default (N) will be to throw an error if non-numeric symbols are found in the string." + }, + { + "name": "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE", + "value": "N", + "description": "System wide flag to ignore timezone while writing date/timestamp value to the database." + }, + { + "name": "HOP_LOG_SIZE_LIMIT", + "value": "0", + "description": "The log size limit for all pipelines and workflows that don't have the \"log size limit\" property set in their respective properties." + }, + { + "name": "HOP_EMPTY_STRING_DIFFERS_FROM_NULL", + "value": "N", + "description": "NULL vs Empty String. If this setting is set to Y, an empty string and null are different. Otherwise they are not." + }, + { + "name": "HOP_MAX_LOG_SIZE_IN_LINES", + "value": "0", + "description": "The maximum number of log lines that are kept internally by Hop. Set to 0 to keep all rows (default)" + }, + { + "name": "HOP_MAX_LOG_TIMEOUT_IN_MINUTES", + "value": "1440", + "description": "The maximum age (in minutes) of a log line while being kept internally by Hop. Set to 0 to keep all rows indefinitely (default)" + }, + { + "name": "HOP_MAX_WORKFLOW_TRACKER_SIZE", + "value": "5000", + "description": "The maximum number of workflow trackers kept in memory" + }, + { + "name": "HOP_MAX_ACTIONS_LOGGED", + "value": "5000", + "description": "The maximum number of action results kept in memory for logging purposes." + }, + { + "name": "HOP_MAX_LOGGING_REGISTRY_SIZE", + "value": "10000", + "description": "The maximum number of logging registry entries kept in memory for logging purposes." + }, + { + "name": "HOP_LOG_TAB_REFRESH_DELAY", + "value": "1000", + "description": "The hop log tab refresh delay." + }, + { + "name": "HOP_LOG_TAB_REFRESH_PERIOD", + "value": "1000", + "description": "The hop log tab refresh period." + }, + { + "name": "HOP_PLUGIN_CLASSES", + "value": null, + "description": "A comma delimited list of classes to scan for plugin annotations" + }, + { + "name": "HOP_PLUGIN_PACKAGES", + "value": null, + "description": "A comma delimited list of packages to scan for plugin annotations (warning: slow!!)" + }, + { + "name": "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT", + "value": "0", + "description": "The maximum number of transform performance snapshots to keep in memory. Set to 0 to keep all snapshots indefinitely (default)" + }, + { + "name": "HOP_ROWSET_GET_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset get timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_ROWSET_PUT_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset put timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_CORE_TRANSFORMS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-transforms.xml file. You can use this to customize the list of available internal transforms outside of the codebase." + }, + { + "name": "HOP_CORE_WORKFLOW_ACTIONS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-workflow-actions.xml file." + }, + { + "name": "HOP_SERVER_OBJECT_TIMEOUT_MINUTES", + "value": "1440", + "description": "This project variable will set a time-out after which waiting, completed or stopped pipelines and workflows will be automatically cleaned up. The default value is 1440 (one day)." + }, + { + "name": "HOP_PIPELINE_PAN_JVM_EXIT_CODE", + "value": null, + "description": "Set this variable to an integer that will be returned as the Pan JVM exit code." + }, + { + "name": "HOP_DISABLE_CONSOLE_LOGGING", + "value": "N", + "description": "Set this variable to Y to disable standard Hop logging to the console. (stdout)" + }, + { + "name": "HOP_REDIRECT_STDERR", + "value": "N", + "description": "Set this variable to Y to redirect stderr to Hop logging." + }, + { + "name": "HOP_REDIRECT_STDOUT", + "value": "N", + "description": "Set this variable to Y to redirect stdout to Hop logging." + }, + { + "name": "HOP_DEFAULT_NUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default number format" + }, + { + "name": "HOP_DEFAULT_BIGNUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default bignumber format" + }, + { + "name": "HOP_DEFAULT_INTEGER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default integer format" + }, + { + "name": "HOP_DEFAULT_DATE_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default date format" + }, + { + "name": "HOP_DEFAULT_TIMESTAMP_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default timestamp format" + }, + { + "name": "HOP_DEFAULT_SERVLET_ENCODING", + "value": null, + "description": "Defines the default encoding for servlets, leave it empty to use Java default encoding" + }, + { + "name": "HOP_FAIL_ON_LOGGING_ERROR", + "value": "N", + "description": "Set this variable to Y when you want the workflow/pipeline fail with an error when the related logging process (e.g. to a database) fails." + }, + { + "name": "HOP_AGGREGATION_MIN_NULL_IS_VALUED", + "value": "N", + "description": "Set this variable to Y to set the minimum to NULL if NULL is within an aggregate. Otherwise by default NULL is ignored by the MIN aggregate and MIN is set to the minimum value that is not NULL. See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO." + }, + { + "name": "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO", + "value": "N", + "description": "Set this variable to Y to return 0 when all values within an aggregate are NULL. Otherwise by default a NULL is returned when all values are NULL." + }, + { + "name": "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Text File Output transform. Setting this to Ywill add no header row at all when the append option is enabled, regardless if the file is existing or not." + }, + { + "name": "HOP_PASSWORD_ENCODER_PLUGIN", + "value": "Hop", + "description": "Specifies the password encoder plugin to use by ID (Hop is the default)." + }, + { + "name": "HOP_SYSTEM_HOSTNAME", + "value": null, + "description": "You can use this variable to speed up hostname lookup. Hostname lookup is performed by Hop so that it is capable of logging the server on which a workflow or pipeline is executed." + }, + { + "name": "HOP_SERVER_JETTY_ACCEPTORS", + "value": null, + "description": "A variable to configure jetty option: acceptors for Carte" + }, + { + "name": "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE", + "value": null, + "description": "A variable to configure jetty option: acceptQueueSize for Carte" + }, + { + "name": "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME", + "value": null, + "description": "A variable to configure jetty option: lowResourcesMaxIdleTime for Carte" + }, + { + "name": "HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Merge Rows (diff) transform. Setting this to Y will use the data from the reference stream (instead of the comparison stream) in case the compared rows are identical." + }, + { + "name": "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE", + "value": "false", + "description": "Set this variable to false to preserve enclosure symbol after splitting the string in the Split fields transform. Changing it to true will remove first and last enclosure symbol from the resulting string chunks." + }, + { + "name": "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES", + "value": "false", + "description": "Set this variable to TRUE to allow your pipeline to pass 'null' fields and/or empty types." + }, + { + "name": "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT", + "value": "false", + "description": "Set this variable to false to preserve global log variables defined in pipeline / workflow Properties -> Log panel. Changing it to true will clear it when export pipeline / workflow." + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_COUNT", + "value": "1024", + "description": "This project variable is used by the Text File Output transform. It defines the max number of simultaneously open files within the transform. The transform will close/reopen files as necessary to insure the max is not exceeded" + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_LIFE", + "value": "0", + "description": "This project variable is used by the Text File Output transform. It defines the max number of milliseconds between flushes of files opened by the transform." + }, + { + "name": "HOP_USE_NATIVE_FILE_DIALOG", + "value": "N", + "description": "Set this value to Y if you want to use the system file open/save dialog when browsing files" + }, + { + "name": "HOP_AUTO_CREATE_CONFIG", + "value": "Y", + "description": "Set this value to N if you don't want to automatically create a hop configuration file (hop-config.json) when it's missing" + } + ], + "LocaleDefault": "en_BE", + "guiProperties": { + "FontFixedSize": "13", + "MaxUndo": "100", + "DarkMode": "Y", + "FontNoteSize": "13", + "ShowOSLook": "Y", + "FontFixedStyle": "0", + "FontNoteName": ".AppleSystemUIFont", + "FontFixedName": "Monospaced", + "FontGraphStyle": "0", + "FontDefaultSize": "13", + "GraphColorR": "255", + "FontGraphSize": "13", + "IconSize": "32", + "BackgroundColorB": "255", + "FontNoteStyle": "0", + "FontGraphName": ".AppleSystemUIFont", + "FontDefaultName": ".AppleSystemUIFont", + "GraphColorG": "255", + "UseGlobalFileBookmarks": "Y", + "FontDefaultStyle": "0", + "GraphColorB": "255", + "BackgroundColorR": "255", + "BackgroundColorG": "255", + "WorkflowDialogStyle": "RESIZE,MAX,MIN", + "LineWidth": "1", + "ContextDialogShowCategories": "Y" + }, + "projectsConfig": { + "enabled": true, + "projectMandatory": true, + "environmentMandatory": false, + "defaultProject": "default", + "defaultEnvironment": null, + "standardParentProject": "default", + "standardProjectsFolder": null, + "projectConfigurations": [ + { + "projectName": "default", + "projectHome": "${HOP_CONFIG_FOLDER}", + "configFilename": "project-config.json" + } + ], + "lifecycleEnvironments": [ + { + "name": "dev", + "purpose": "Testing", + "projectName": "default", + "configurationFiles": [ + "${PROJECT_HOME}/dev-env-config.json" + ] + } + ], + "projectLifecycles": [] + } +} \ No newline at end of file diff --git a/integration-tests/salesforce/main-0001-salesforce-bulk.hwf b/integration-tests/salesforce/main-0001-salesforce-bulk.hwf new file mode 100644 index 00000000000..cfab6a4268d --- /dev/null +++ b/integration-tests/salesforce/main-0001-salesforce-bulk.hwf @@ -0,0 +1,194 @@ + + + + main-0001-salesforce-bulk + Y + + + 0 + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 48 + 48 + + + + 0001-bulk-insert.hpl + + PIPELINE + + ${PROJECT_HOME}/0001-bulk-insert.hpl + Basic + + Y + + local + Y + N + 200 + 48 + + + + 0002-bulk-insert-error.hpl + + PIPELINE + + ${PROJECT_HOME}/0002-bulk-insert-error.hpl + Basic + + Y + + local + Y + N + 360 + 48 + + + + 0003-bulk-query.hpl + + PIPELINE + + ${PROJECT_HOME}/0003-bulk-query.hpl + Basic + + Y + + local + Y + N + 520 + 48 + + + + 0004-bulk-update.hpl + + PIPELINE + + ${PROJECT_HOME}/0004-bulk-update.hpl + Basic + + Y + + local + Y + N + 680 + 48 + + + + 0005-bulk-delete.hpl + + PIPELINE + + ${PROJECT_HOME}/0005-bulk-delete.hpl + Basic + + Y + + local + Y + N + 840 + 48 + + + + 0006-bulk-upsert.hpl + + PIPELINE + + ${PROJECT_HOME}/0006-bulk-upsert.hpl + Basic + + Y + + local + Y + N + 1000 + 48 + + + + + + Start + 0001-bulk-insert.hpl + Y + Y + Y + + + 0001-bulk-insert.hpl + 0002-bulk-insert-error.hpl + Y + Y + N + + + 0002-bulk-insert-error.hpl + 0003-bulk-query.hpl + Y + Y + N + + + 0003-bulk-query.hpl + 0004-bulk-update.hpl + Y + Y + N + + + 0004-bulk-update.hpl + 0005-bulk-delete.hpl + Y + Y + N + + + 0005-bulk-delete.hpl + 0006-bulk-upsert.hpl + Y + Y + N + + + + diff --git a/integration-tests/salesforce/metadata/pipeline-run-configuration/local.json b/integration-tests/salesforce/metadata/pipeline-run-configuration/local.json new file mode 100644 index 00000000000..63794efcaf6 --- /dev/null +++ b/integration-tests/salesforce/metadata/pipeline-run-configuration/local.json @@ -0,0 +1,17 @@ +{ + "engineRunConfiguration": { + "Local": { + "feedback_size": "50000", + "sample_size": "100", + "sample_type_in_gui": "Last", + "rowset_size": "10000", + "safe_mode": false, + "show_feedback": false, + "topo_sort": false, + "gather_metrics": false + } + }, + "configurationVariables": [], + "name": "local", + "description": "Runs your pipelines locally with the standard local Hop pipeline engine" +} \ No newline at end of file diff --git a/integration-tests/salesforce/metadata/salesforce-connection/salesforce-mock.json b/integration-tests/salesforce/metadata/salesforce-connection/salesforce-mock.json new file mode 100644 index 00000000000..e5c2ca0a022 --- /dev/null +++ b/integration-tests/salesforce/metadata/salesforce-connection/salesforce-mock.json @@ -0,0 +1,8 @@ +{ + "name": "salesforce-mock", + "authentication_type": "OAUTH", + "oauth_client_id": "integration-test", + "oauth_access_token": "test-token", + "oauth_instance_url": "http://salesforce_mock:8080", + "oauth_api_version": "64.0" +} diff --git a/integration-tests/salesforce/metadata/workflow-run-configuration/local.json b/integration-tests/salesforce/metadata/workflow-run-configuration/local.json new file mode 100644 index 00000000000..e37a93039aa --- /dev/null +++ b/integration-tests/salesforce/metadata/workflow-run-configuration/local.json @@ -0,0 +1,9 @@ +{ + "engineRunConfiguration": { + "Local": { + "safe_mode": false + } + }, + "name": "local", + "description": "Runs your workflows locally with the standard local Hop workflow engine" +} \ No newline at end of file diff --git a/integration-tests/salesforce/project-config.json b/integration-tests/salesforce/project-config.json new file mode 100644 index 00000000000..899c6c10927 --- /dev/null +++ b/integration-tests/salesforce/project-config.json @@ -0,0 +1,15 @@ +{ + "metadataBaseFolder": "${PROJECT_HOME}/metadata", + "unitTestsBasePath": "${PROJECT_HOME}", + "dataSetsCsvFolder": "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome": true, + "config": { + "variables": [ + { + "name": "HOP_LICENSE_HEADER_FILE", + "value": "${PROJECT_HOME}/../asf-header.txt", + "description": "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" + } + ] + } +} \ No newline at end of file diff --git a/integration-tests/samples/read-samples-build-hop-run.hpl b/integration-tests/samples/read-samples-build-hop-run.hpl index a5c81d9c650..c12e0fc36d9 100644 --- a/integration-tests/samples/read-samples-build-hop-run.hpl +++ b/integration-tests/samples/read-samples-build-hop-run.hpl @@ -739,6 +739,14 @@ var hop_run_cmd = './hop-run.sh -j samples -r ' + run_config + ' -f ' + filename salesforce-delete.hpl requires Salesforce credentials + + salesforce-bulk-load.hpl + requires Salesforce credentials + + + salesforce-bulk-input.hpl + requires Salesforce credentials + pgvector-search.hpl requires PostgreSQL with the pgvector extension diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java index 79d7b784fef..fedc87b704e 100644 --- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java +++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java @@ -131,7 +131,11 @@ public class HopPipelineMetaToBeamPipelineConverter { public static final Map HARD_BANNED_PLUGIN_IDS = Map.of( BeamConst.STRING_MASK_FIELDS_PLUGIN_ID, - "Mask fields is not supported on Beam. Each worker keeps its own mapping and sequence, so the same source value would not stay the same token."); + "Mask fields is not supported on Beam. Each worker keeps its own mapping and sequence, so the same source value would not stay the same token.", + BeamConst.STRING_SALESFORCE_BULK_LOAD_PLUGIN_ID, + "Salesforce bulk load is not supported on Beam. The transform buffers rows and sends them as one Bulk API job, so splitting the rows across workers would open separate jobs.", + BeamConst.STRING_SALESFORCE_BULK_INPUT_PLUGIN_ID, + "Salesforce bulk input is not supported on Beam. The transform runs one query job and reads the result pages in order."); protected final String runConfigName; protected final PipelineRunConfiguration runConfiguration; diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java index e041851c620..4df1615d8f0 100644 --- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java +++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java @@ -53,6 +53,14 @@ public class BeamConst { /** Mask fields. Banned by plugin id so the Beam engine does not have to load the transform. */ public static final String STRING_MASK_FIELDS_PLUGIN_ID = "MaskFields"; + /** + * Salesforce bulk transforms. Banned by plugin id so the Beam engine does not load the Salesforce + * plugin. + */ + public static final String STRING_SALESFORCE_BULK_LOAD_PLUGIN_ID = "SalesforceBulkLoad"; + + public static final String STRING_SALESFORCE_BULK_INPUT_PLUGIN_ID = "SalesforceBulkInput"; + public static final String STRING_HOP_BEAM = "Hop Beam"; public static final String STRING_TRANSFORM_FLAG_BATCH = "Batch"; diff --git a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java index 10d432d1893..f8aa117b97c 100644 --- a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java +++ b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java @@ -72,6 +72,23 @@ void uniqueRowsByHashSetMetaIsHardBanned() { assertTrue(engine.supports(pluginWithMainType(UniqueRowsByHashSetMeta.class)).isUnsupported()); } + @Test + void salesforceBulkPluginIdsAreHardBanned() { + // Ban by plugin id. Referencing the Salesforce classes here would put them on the Beam engine + // class loader. + for (String id : + new String[] { + BeamConst.STRING_SALESFORCE_BULK_LOAD_PLUGIN_ID, + BeamConst.STRING_SALESFORCE_BULK_INPUT_PLUGIN_ID + }) { + EngineCompatibility verdict = engine.supports(pluginWithIdAndMainType(id, Object.class)); + assertTrue(verdict.isUnsupported(), id); + assertEquals( + HopPipelineMetaToBeamPipelineConverter.HARD_BANNED_PLUGIN_IDS.get(id), + verdict.getReason()); + } + } + @Test void maskFieldsPluginIdIsHardBanned() { // Ban by plugin id. Referencing MaskFieldsMeta here would put that class on the Beam engine diff --git a/plugins/engines/spark/pom.xml b/plugins/engines/spark/pom.xml index 4d84b67ecc5..32dac7170dc 100644 --- a/plugins/engines/spark/pom.xml +++ b/plugins/engines/spark/pom.xml @@ -302,6 +302,12 @@ h2 test + + org.apache.hop + hop-tech-salesforce + ${project.version} + test + org.apache.hop diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java index 68265699d6d..e86098e1e9a 100644 --- a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java +++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java @@ -100,7 +100,11 @@ public class HopPipelineMetaToSparkConverter { SparkConst.JOIN_ROWS_PLUGIN_ID, "Join Rows is not supported on the native Spark engine. A cartesian product needs every row of every input in one place, but every partition would only combine the rows it happens to hold, so combinations would go missing. Add the same constant field to both inputs and use Merge Join on that field instead.", SparkConst.MASK_FIELDS_PLUGIN_ID, - "Mask fields is not supported on the native Spark engine. Every partition keeps its own mapping and sequence, so the same source value would not stay the same token."); + "Mask fields is not supported on the native Spark engine. Every partition keeps its own mapping and sequence, so the same source value would not stay the same token.", + SparkConst.SALESFORCE_BULK_LOAD_PLUGIN_ID, + "Salesforce bulk load is not supported on the native Spark engine. The transform buffers rows and sends them as one Bulk API job, so each partition would open its own job.", + SparkConst.SALESFORCE_BULK_INPUT_PLUGIN_ID, + "Salesforce bulk input is not supported on the native Spark engine. The transform runs one query job and reads the result pages in order."); private final IVariables variables; private final PipelineMeta pipelineMeta; diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/util/SparkConst.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/util/SparkConst.java index 2d72ae47319..1ebe0a2c178 100644 --- a/plugins/engines/spark/src/main/java/org/apache/hop/spark/util/SparkConst.java +++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/util/SparkConst.java @@ -42,6 +42,8 @@ public final class SparkConst { public static final String GROUP_BY_PLUGIN_ID = "GroupBy"; public static final String JOIN_ROWS_PLUGIN_ID = "JoinRows"; public static final String MASK_FIELDS_PLUGIN_ID = "MaskFields"; + public static final String SALESFORCE_BULK_LOAD_PLUGIN_ID = "SalesforceBulkLoad"; + public static final String SALESFORCE_BULK_INPUT_PLUGIN_ID = "SalesforceBulkInput"; public static final String SPARK_FILE_INPUT_PLUGIN_ID = "SparkFileInput"; public static final String SPARK_FILE_OUTPUT_PLUGIN_ID = "SparkFileOutput"; diff --git a/plugins/engines/spark/src/test/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverterTest.java b/plugins/engines/spark/src/test/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverterTest.java index f919cc2e34d..28fabd13177 100644 --- a/plugins/engines/spark/src/test/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverterTest.java +++ b/plugins/engines/spark/src/test/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverterTest.java @@ -40,6 +40,8 @@ import org.apache.hop.pipeline.transforms.groupby.GroupByMeta; import org.apache.hop.pipeline.transforms.joinrows.JoinRowsMeta; import org.apache.hop.pipeline.transforms.maskfields.MaskFieldsMeta; +import org.apache.hop.pipeline.transforms.salesforcebulk.SalesforceBulkInputMeta; +import org.apache.hop.pipeline.transforms.salesforcebulk.SalesforceBulkLoadMeta; import org.apache.hop.pipeline.transforms.uniquerowsbyhashset.UniqueRowsByHashSetMeta; import org.apache.hop.spark.engines.SparkPipelineEngine; import org.apache.hop.spark.util.SparkConst; @@ -183,7 +185,11 @@ void hardBannedTransformsExcludeNativeSparkOnAnnotation() { SparkConst.JOIN_ROWS_PLUGIN_ID, JoinRowsMeta.class, SparkConst.MASK_FIELDS_PLUGIN_ID, - MaskFieldsMeta.class); + MaskFieldsMeta.class, + SparkConst.SALESFORCE_BULK_LOAD_PLUGIN_ID, + SalesforceBulkLoadMeta.class, + SparkConst.SALESFORCE_BULK_INPUT_PLUGIN_ID, + SalesforceBulkInputMeta.class); assertEquals( HopPipelineMetaToSparkConverter.HARD_BANNED_PLUGIN_IDS.keySet(), diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkClient.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkClient.java new file mode 100644 index 00000000000..b63178989b4 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkClient.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.core.exception.HopException; + +/** Bulk API 2.0 calls used by the load and input transforms. */ +public interface SalesforceBulkClient extends AutoCloseable { + SalesforceBulkJob createIngestJob(SalesforceBulkIngestRequest request) throws HopException; + + void uploadIngestData(String jobId, byte[] csv) throws HopException; + + SalesforceBulkJob completeIngestUpload(String jobId) throws HopException; + + SalesforceBulkJob getIngestJob(String jobId) throws HopException; + + /** Successful results when {@code failed} is false, failed results otherwise. */ + String ingestResults(String jobId, boolean failed) throws HopException; + + void abortIngestJob(String jobId) throws HopException; + + SalesforceBulkJob createQueryJob(String operation, String soql) throws HopException; + + SalesforceBulkJob getQueryJob(String jobId) throws HopException; + + SalesforceBulkQueryPage queryResults(String jobId, String locator) throws HopException; + + void abortQueryJob(String jobId) throws HopException; + + @Override + void close(); +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkConcurrency.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkConcurrency.java new file mode 100644 index 00000000000..6d2167808f8 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkConcurrency.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +/** Bulk API 2.0 ingest concurrency. Serial avoids row-lock errors on a shared parent. */ +public enum SalesforceBulkConcurrency implements IEnumHasCodeAndDescription { + PARALLEL("Parallel", "SalesforceBulkConcurrency.Parallel"), + SERIAL("Serial", "SalesforceBulkConcurrency.Serial"); + + private final String code; + private final String descriptionKey; + + SalesforceBulkConcurrency(String code, String descriptionKey) { + this.code = code; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return BaseMessages.getString(SalesforceBulkConcurrency.class, descriptionKey); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsv.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsv.java new file mode 100644 index 00000000000..49704700045 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsv.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.exception.HopException; + +/** CSV rules for Bulk API 2.0: quote when required, and {@code #N/A} for a null on update. */ +public final class SalesforceBulkCsv { + /** Value written for a null on update and upsert. An empty field does not clear a value. */ + public static final String NULL_TOKEN = "#N/A"; + + public static final String SUCCESS_ID = "sf__Id"; + public static final String SUCCESS_CREATED = "sf__Created"; + public static final String FAILURE_ERROR = "sf__Error"; + + private SalesforceBulkCsv() {} + + public static String encode(String value, String delimiter) { + if (value == null) { + return ""; + } + boolean quote = + value.indexOf(delimiter) >= 0 + || value.indexOf('"') >= 0 + || value.indexOf('\n') >= 0 + || value.indexOf('\r') >= 0 + || value.startsWith(" ") + || value.endsWith(" "); + String escaped = value.replace("\"", "\"\""); + if (quote) { + return "\"" + escaped + "\""; + } + return escaped; + } + + public static byte[] toBytes(String csv) { + return csv.getBytes(StandardCharsets.UTF_8); + } + + /** Parsed table. The first row is the header. */ + public static List> parse(String csv) throws HopException { + return parse(csv, SalesforceBulkDelimiter.COMMA); + } + + public static List> parse(String csv, SalesforceBulkDelimiter columnDelimiter) + throws HopException { + String delimiter = columnDelimiter == null ? "," : columnDelimiter.getCharacter(); + List> rows = new ArrayList<>(); + if (csv == null || csv.isEmpty()) { + return rows; + } + List row = new ArrayList<>(); + StringBuilder field = new StringBuilder(); + boolean quoted = false; + for (int i = 0; i < csv.length(); i++) { + char current = csv.charAt(i); + if (quoted) { + if (current == '"') { + if (i + 1 < csv.length() && csv.charAt(i + 1) == '"') { + field.append('"'); + i++; + } else { + quoted = false; + } + } else { + field.append(current); + } + continue; + } + if (current == '"' && field.length() == 0) { + quoted = true; + continue; + } + if (delimiter != null && current == delimiter.charAt(0) && delimiter.length() == 1) { + row.add(field.toString()); + field.setLength(0); + continue; + } + if (current == '\n' || current == '\r') { + row.add(field.toString()); + field.setLength(0); + if (!isBlankRow(row)) { + rows.add(row); + } + row = new ArrayList<>(); + if (current == '\r' && i + 1 < csv.length() && csv.charAt(i + 1) == '\n') { + i++; + } + continue; + } + field.append(current); + } + if (quoted) { + throw new HopException("A Bulk API result CSV has an unterminated quote"); + } + if (field.length() > 0 || !row.isEmpty()) { + row.add(field.toString()); + if (!isBlankRow(row)) { + rows.add(row); + } + } + return rows; + } + + private static boolean isBlankRow(List row) { + for (String value : row) { + if (value != null && !value.isEmpty()) { + return false; + } + } + return true; + } + + /** One CSV document built with a fixed header, delimiter, and line ending. */ + public static final class Builder { + private final List headers; + private final String delimiter; + private final String lineEnding; + private final StringBuilder csv = new StringBuilder(); + private int rows; + + public Builder( + List headers, + SalesforceBulkDelimiter delimiter, + SalesforceBulkLineEnding lineEnding) { + this.headers = List.copyOf(headers); + this.delimiter = delimiter.getCharacter(); + this.lineEnding = lineEnding.getCharacters(); + writeRow(this.headers); + } + + public int sizeBytes() { + return csv.toString().getBytes(StandardCharsets.UTF_8).length; + } + + public int getRows() { + return rows; + } + + public void add(List values) { + writeRow(values); + rows++; + } + + public byte[] toByteArray() { + return csv.toString().getBytes(StandardCharsets.UTF_8); + } + + private void writeRow(List values) { + for (int i = 0; i < values.size(); i++) { + if (i > 0) { + csv.append(delimiter); + } + csv.append(encode(values.get(i) == null ? "" : values.get(i), delimiter)); + } + csv.append(lineEnding); + } + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkDelimiter.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkDelimiter.java new file mode 100644 index 00000000000..5ac6d30a8b0 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkDelimiter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +/** Column delimiter declared on a Bulk API 2.0 ingest job. */ +public enum SalesforceBulkDelimiter implements IEnumHasCodeAndDescription { + COMMA("COMMA", ",", "SalesforceBulkDelimiter.Comma"), + SEMICOLON("SEMICOLON", ";", "SalesforceBulkDelimiter.Semicolon"), + TAB("TAB", "\t", "SalesforceBulkDelimiter.Tab"), + PIPE("PIPE", "|", "SalesforceBulkDelimiter.Pipe"), + CARET("CARET", "^", "SalesforceBulkDelimiter.Caret"), + BACKQUOTE("BACKQUOTE", "`", "SalesforceBulkDelimiter.Backquote"); + + private final String code; + private final String character; + private final String descriptionKey; + + SalesforceBulkDelimiter(String code, String character, String descriptionKey) { + this.code = code; + this.character = character; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + public String getCharacter() { + return character; + } + + @Override + public String getDescription() { + return BaseMessages.getString(SalesforceBulkDelimiter.class, descriptionKey); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkField.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkField.java new file mode 100644 index 00000000000..cfbb43f6c24 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkField.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.gui.plugin.GuiTableColumn; +import org.apache.hop.core.gui.plugin.GuiTableColumnType; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** One stream field written as a Salesforce field in a bulk load. */ +@Getter +@Setter +public class SalesforceBulkField { + @HopMetadataProperty(key = "stream") + @GuiTableColumn( + id = "stream", + order = "10", + type = GuiTableColumnType.TEXT, + label = "i18n::SalesforceBulkField.Stream.Label", + width = 220) + private String streamField = ""; + + @HopMetadataProperty(key = "salesforce") + @GuiTableColumn( + id = "salesforce", + order = "20", + type = GuiTableColumnType.TEXT, + label = "i18n::SalesforceBulkField.Salesforce.Label", + width = 220) + private String salesforceField = ""; + + public SalesforceBulkField() {} + + public SalesforceBulkField(String streamField, String salesforceField) { + this.streamField = streamField; + this.salesforceField = salesforceField; + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClient.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClient.java new file mode 100644 index 00000000000..61e6f895782 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClient.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPatch; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; +import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.util.HttpClientManager; +import org.apache.hop.core.util.Utils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +/** Bulk API 2.0 over {@link HttpClientManager}. */ +public class SalesforceBulkHttpClient implements SalesforceBulkClient { + private static final ContentType CSV = ContentType.create("text/csv", StandardCharsets.UTF_8); + + private final SalesforceBulkSession session; + private final CloseableHttpClient httpClient; + private final String jobsRoot; + + public SalesforceBulkHttpClient(SalesforceBulkSession session) { + this(session, HttpClientManager.getInstance().createBuilder().useSystemProxy(true).build()); + } + + SalesforceBulkHttpClient(SalesforceBulkSession session, CloseableHttpClient httpClient) { + this.session = session; + this.httpClient = httpClient; + this.jobsRoot = + session.getInstanceUrl() + "/services/data/v" + session.getApiVersion() + "/jobs"; + } + + @Override + public SalesforceBulkJob createIngestJob(SalesforceBulkIngestRequest request) + throws HopException { + JSONObject body = new JSONObject(); + body.put("object", request.getSalesforceObject()); + body.put("operation", request.getOperation().getCode()); + body.put("contentType", "CSV"); + body.put("lineEnding", request.getLineEnding().getCode()); + body.put("columnDelimiter", request.getDelimiter().getCode()); + body.put("concurrencyMode", request.getConcurrency().getCode()); + if (request.getOperation() == SalesforceBulkOperation.UPSERT + && !Utils.isEmpty(request.getExternalIdField())) { + body.put("externalIdFieldName", request.getExternalIdField()); + } + HttpPost post = new HttpPost(jobsRoot + "/ingest"); + post.setEntity(new StringEntity(body.toJSONString(), ContentType.APPLICATION_JSON)); + return readJob(exchange(post, false)); + } + + @Override + public void uploadIngestData(String jobId, byte[] csv) throws HopException { + HttpPut put = new HttpPut(jobsRoot + "/ingest/" + jobId + "/batches"); + put.setEntity(new ByteArrayEntity(csv, CSV)); + exchange(put, false); + } + + @Override + public SalesforceBulkJob completeIngestUpload(String jobId) throws HopException { + return patchJob(jobsRoot + "/ingest/" + jobId, "UploadComplete"); + } + + @Override + public SalesforceBulkJob getIngestJob(String jobId) throws HopException { + return readJob(exchange(new HttpGet(jobsRoot + "/ingest/" + jobId), false)); + } + + @Override + public String ingestResults(String jobId, boolean failed) throws HopException { + String suffix = failed ? "/failedResults" : "/successfulResults"; + HttpGet get = new HttpGet(jobsRoot + "/ingest/" + jobId + suffix); + return exchange(get, true).body; + } + + @Override + public void abortIngestJob(String jobId) throws HopException { + patchJob(jobsRoot + "/ingest/" + jobId, "Aborted"); + } + + @Override + public SalesforceBulkJob createQueryJob(String operation, String soql) throws HopException { + JSONObject body = new JSONObject(); + body.put("operation", operation); + body.put("query", soql); + HttpPost post = new HttpPost(jobsRoot + "/query"); + post.setEntity(new StringEntity(body.toJSONString(), ContentType.APPLICATION_JSON)); + return readJob(exchange(post, false)); + } + + @Override + public SalesforceBulkJob getQueryJob(String jobId) throws HopException { + return readJob(exchange(new HttpGet(jobsRoot + "/query/" + jobId), false)); + } + + @Override + public SalesforceBulkQueryPage queryResults(String jobId, String locator) throws HopException { + HttpGet get = new HttpGet(jobsRoot + "/query/" + jobId + "/results"); + if (locator != null && !locator.isBlank() && !"null".equalsIgnoreCase(locator)) { + get.setHeader("Sforce-Locator", locator); + } + HttpResult result = exchange(get, true); + return new SalesforceBulkQueryPage(result.body, result.locator); + } + + @Override + public void abortQueryJob(String jobId) throws HopException { + patchJob(jobsRoot + "/query/" + jobId, "Aborted"); + } + + @Override + public void close() { + try { + httpClient.close(); + } catch (IOException ignored) { + // The shared connection pool stays open. + } + } + + private SalesforceBulkJob patchJob(String url, String state) throws HopException { + JSONObject body = new JSONObject(); + body.put("state", state); + HttpPatch patch = new HttpPatch(url); + patch.setEntity(new StringEntity(body.toJSONString(), ContentType.APPLICATION_JSON)); + return readJob(exchange(patch, false)); + } + + private HttpResult exchange(HttpUriRequestBase request, boolean allowMissing) + throws HopException { + request.setHeader("Authorization", "Bearer " + session.getAccessToken()); + request.setHeader("Accept", "application/json, text/csv"); + try { + return httpClient.execute( + request, + response -> { + String body = + response.getEntity() == null + ? "" + : EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + Header locator = response.getFirstHeader("Sforce-Locator"); + int code = response.getCode(); + if (allowMissing && code == 404) { + return new HttpResult(code, "", null); + } + if (code >= 300) { + throw new java.io.IOException( + "Salesforce Bulk API " + request.getMethod() + " returned " + code + ": " + body); + } + return new HttpResult(code, body, locator == null ? null : locator.getValue()); + }); + } catch (Exception e) { + String message = e.getMessage() == null ? e.toString() : e.getMessage(); + throw new HopException(message, e); + } + } + + private static SalesforceBulkJob readJob(HttpResult result) throws HopException { + try { + Object parsed = new JSONParser().parse(result.body); + if (!(parsed instanceof JSONObject object)) { + throw new HopException("Salesforce Bulk API returned no job: " + result.body); + } + return new SalesforceBulkJob( + text(object, "id"), + text(object, "state"), + text(object, "errorMessage"), + integer(object, "numberRecordsProcessed"), + integer(object, "numberRecordsFailed")); + } catch (ParseException e) { + throw new HopException("Salesforce Bulk API returned invalid JSON: " + result.body, e); + } + } + + private static String text(JSONObject object, String key) { + Object value = object.get(key); + return value == null ? null : String.valueOf(value); + } + + private static int integer(JSONObject object, String key) { + Object value = object.get(key); + if (value instanceof Number number) { + return number.intValue(); + } + if (value == null || String.valueOf(value).isBlank()) { + return 0; + } + return Integer.parseInt(String.valueOf(value)); + } + + private static final class HttpResult { + private final int code; + private final String body; + private final String locator; + + private HttpResult(int code, String body, String locator) { + this.code = code; + this.body = body == null ? "" : body; + this.locator = locator; + } + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkIngestRequest.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkIngestRequest.java new file mode 100644 index 00000000000..60e896ace77 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkIngestRequest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import lombok.Getter; + +/** Fields sent when a Bulk API 2.0 ingest job is created. */ +@Getter +public class SalesforceBulkIngestRequest { + private final String salesforceObject; + private final SalesforceBulkOperation operation; + private final String externalIdField; + private final SalesforceBulkLineEnding lineEnding; + private final SalesforceBulkDelimiter delimiter; + private final SalesforceBulkConcurrency concurrency; + + public SalesforceBulkIngestRequest( + String salesforceObject, + SalesforceBulkOperation operation, + String externalIdField, + SalesforceBulkLineEnding lineEnding, + SalesforceBulkDelimiter delimiter, + SalesforceBulkConcurrency concurrency) { + this.salesforceObject = salesforceObject; + this.operation = operation; + this.externalIdField = externalIdField; + this.lineEnding = lineEnding == null ? SalesforceBulkLineEnding.LF : lineEnding; + this.delimiter = delimiter == null ? SalesforceBulkDelimiter.COMMA : delimiter; + this.concurrency = concurrency == null ? SalesforceBulkConcurrency.PARALLEL : concurrency; + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInput.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInput.java new file mode 100644 index 00000000000..37074eb02e3 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInput.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** Reads a SOQL query through a Bulk API 2.0 query job. */ +public class SalesforceBulkInput + extends BaseTransform { + private static final Class PKG = SalesforceBulkInput.class; + + public SalesforceBulkInput( + TransformMeta transformMeta, + SalesforceBulkInputMeta meta, + SalesforceBulkInputData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + if (data.client != null) { + return prepareSettings(); + } + try { + data.client = + SalesforceBulkRuntime.open(this, metadataProvider, getLogChannel(), meta.getConnection()); + return prepareSettings(); + } catch (Exception e) { + logError(e.getMessage(), e); + return false; + } + } + + private boolean prepareSettings() { + try { + data.outputRowMeta = new org.apache.hop.core.row.RowMeta(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + data.columnNames = new ArrayList<>(); + data.targets = new ArrayList<>(); + if (meta.getFields() != null) { + for (SalesforceBulkInputField field : meta.getFields()) { + if (field == null || Utils.isEmpty(field.getName())) { + continue; + } + String name = resolve(field.getName()); + data.columnNames.add(name); + data.targets.add(data.outputRowMeta.searchValueMeta(name)); + } + } + if (data.columnNames.isEmpty()) { + logError(BaseMessages.getString(PKG, "SalesforceBulkInput.Error.NoFields")); + return false; + } + String jobField = resolve(meta.getJobIdField()); + data.jobIdIndex = Utils.isEmpty(jobField) ? -1 : data.outputRowMeta.indexOfValue(jobField); + data.timeoutMillis = SalesforceBulkRuntime.secondsToMillis(resolve(meta.getTimeout()), 3600); + data.pollMillis = SalesforceBulkRuntime.secondsToMillis(resolve(meta.getPollInterval()), 10); + data.maxRows = longValue(resolve(meta.getMaxRows())); + SalesforceBulkQueryOperation operation = + meta.getOperation() == null ? SalesforceBulkQueryOperation.QUERY : meta.getOperation(); + data.operationCode = operation.getCode(); + data.soql = resolve(meta.getSoql()); + if (Utils.isEmpty(data.soql)) { + logError(BaseMessages.getString(PKG, "SalesforceBulkInput.Error.NoSoql")); + return false; + } + return true; + } catch (Exception e) { + logError(e.getMessage(), e); + return false; + } + } + + @Override + public boolean processRow() throws HopException { + if (!data.started) { + startQuery(); + } + if (data.pageIndex >= data.page.size() && !loadNextPage()) { + setOutputDone(); + return false; + } + if (data.maxRows > 0 && getLinesOutput() >= data.maxRows) { + setOutputDone(); + return false; + } + putRow(data.outputRowMeta, data.page.get(data.pageIndex++)); + return true; + } + + private void startQuery() throws HopException { + data.started = true; + SalesforceBulkJob created = data.client.createQueryJob(data.operationCode, data.soql); + data.openJobId = created.getId(); + data.jobId = created.getId(); + SalesforceBulkJob finished = + SalesforceBulkRuntime.poll( + data.client, + created.getId(), + true, + data.timeoutMillis, + data.pollMillis, + this::isStopped); + data.openJobId = null; + if (finished.isAborted() || "Failed".equals(finished.getState())) { + throw new HopException( + BaseMessages.getString( + PKG, + "SalesforceBulkInput.Error.QueryFailed", + created.getId(), + Utils.isEmpty(finished.getErrorMessage()) + ? finished.getState() + : finished.getErrorMessage())); + } + logBasic(BaseMessages.getString(PKG, "SalesforceBulkInput.Log.JobFinished", created.getId())); + data.locator = null; + } + + private boolean loadNextPage() throws HopException { + if (data.started && data.locator == null && data.pageIndex > 0 && !data.page.isEmpty()) { + return false; + } + if (data.pageIndex > 0 && (data.locator == null || "null".equalsIgnoreCase(data.locator))) { + return false; + } + SalesforceBulkQueryPage page = data.client.queryResults(data.jobId, data.locator); + data.locator = page.hasMore() ? page.getLocator() : null; + data.page = convert(page.getCsv()); + data.pageIndex = 0; + return !data.page.isEmpty(); + } + + private List convert(String csv) throws HopException { + List> table = SalesforceBulkCsv.parse(csv); + List rows = new ArrayList<>(); + if (table.size() < 2) { + return rows; + } + Map index = new HashMap<>(); + List headers = table.get(0); + for (int i = 0; i < headers.size(); i++) { + index.put(headers.get(i), i); + } + for (String column : data.columnNames) { + if (!index.containsKey(column)) { + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulkInput.Error.MissingColumn", column)); + } + } + for (int r = 1; r < table.size(); r++) { + List source = table.get(r); + Object[] row = RowDataUtil.allocateRowData(data.outputRowMeta.size()); + for (int c = 0; c < data.columnNames.size(); c++) { + int at = index.get(data.columnNames.get(c)); + String text = at < source.size() ? source.get(at) : ""; + IValueMeta target = data.targets.get(c); + row[c] = SalesforceBulkValues.parse(target, text); + } + if (data.jobIdIndex >= 0) { + row[data.jobIdIndex] = data.jobId; + } + rows.add(row); + } + return rows; + } + + private static long longValue(String text) { + if (Utils.isEmpty(text)) { + return 0; + } + try { + return Long.parseLong(text.trim()); + } catch (NumberFormatException e) { + return 0; + } + } + + @Override + public void dispose() { + if (data.openJobId != null) { + SalesforceBulkRuntime.abort(data.client, data.openJobId, true); + data.openJobId = null; + } + if (data.client != null) { + data.client.close(); + data.client = null; + } + super.dispose(); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputData.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputData.java new file mode 100644 index 00000000000..73fe42235a2 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputData.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +@SuppressWarnings("java:S1104") +public class SalesforceBulkInputData extends BaseTransformData implements ITransformData { + public SalesforceBulkClient client; + public IRowMeta outputRowMeta; + public List targets = new ArrayList<>(); + public List columnNames = new ArrayList<>(); + public int jobIdIndex = -1; + public String jobId; + public String openJobId; + public String locator; + public boolean started; + public List page = new ArrayList<>(); + public int pageIndex; + public long timeoutMillis = 3_600_000L; + public long pollMillis = 10_000L; + public long maxRows; + public String operationCode; + public String soql; +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputDialog.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputDialog.java new file mode 100644 index 00000000000..8d6402af42c --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputDialog.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.widgets.Shell; + +public class SalesforceBulkInputDialog extends BaseTransformDialog { + private static final Class PKG = SalesforceBulkInputMeta.class; + + private final SalesforceBulkInputMeta input; + private GuiCompositeWidgets widgets; + + public SalesforceBulkInputDialog( + Shell parent, + IVariables variables, + SalesforceBulkInputMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + createShell(BaseMessages.getString(PKG, "SalesforceBulkInput.Name")); + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + widgets = + GuiCompositeWidgets.addScrolledComposite( + shell, + variables, + wTransformName, + wOk, + SalesforceBulkInputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, + input); + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + private void cancel() { + transformName = null; + dispose(); + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + widgets.getWidgetsContents(input, SalesforceBulkInputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + transformName = wTransformName.getText(); + dispose(); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputField.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputField.java new file mode 100644 index 00000000000..66909412f14 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputField.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.gui.plugin.GuiTableColumn; +import org.apache.hop.core.gui.plugin.GuiTableColumnType; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** One column read from a Bulk API 2.0 query result. */ +@Getter +@Setter +public class SalesforceBulkInputField { + @HopMetadataProperty(key = "name") + @GuiTableColumn( + id = "name", + order = "10", + type = GuiTableColumnType.TEXT, + label = "i18n::SalesforceBulkInputField.Name.Label", + width = 220) + private String name = ""; + + @HopMetadataProperty(key = "type") + @GuiTableColumn( + id = "type", + order = "20", + type = GuiTableColumnType.COMBO, + comboValuesMethod = "hopTypeNames", + label = "i18n::SalesforceBulkInputField.Type.Label", + width = 140) + private String type = "String"; + + public SalesforceBulkInputField() {} + + public SalesforceBulkInputField(String name, String type) { + this.name = name; + this.type = type; + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputMeta.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputMeta.java new file mode 100644 index 00000000000..6b95e7251c3 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkInputMeta.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaFactory; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.salesforce.SalesforceConnection; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@GuiPlugin +@Transform( + id = "SalesforceBulkInput", + name = "i18n::SalesforceBulkInput.Name", + description = "i18n::SalesforceBulkInput.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input", + keywords = "i18n::SalesforceBulkInput.keyword", + image = "salesforcebulkinput.svg", + documentationUrl = "/pipeline/transforms/salesforcebulkinput.html", + excludedEngines = {"Beam*", "SparkPipelineEngine"}) +public class SalesforceBulkInputMeta + extends BaseTransformMeta { + private static final Class PKG = SalesforceBulkInputMeta.class; + + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "SalesforceBulkInput.Dialog"; + private static final String GROUP_CONNECTION = "i18n::SalesforceBulkInput.Group.Connection"; + private static final String GROUP_QUERY = "i18n::SalesforceBulkInput.Group.Query"; + private static final String GROUP_FIELDS = "i18n::SalesforceBulkInput.Group.Fields"; + + @GuiWidgetElement( + id = "connection", + order = "100", + type = GuiElementType.METADATA, + metadata = SalesforceConnection.class, + label = "i18n::SalesforceBulkInput.Connection.Label", + toolTip = "i18n::SalesforceBulkInput.Connection.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_CONNECTION, + groupOrder = "100") + @HopMetadataProperty(key = "connection") + private String connection; + + @GuiWidgetElement( + id = "operation", + order = "100", + type = GuiElementType.COMBO, + label = "i18n::SalesforceBulkInput.Operation.Label", + toolTip = "i18n::SalesforceBulkInput.Operation.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "operation", storeWithCode = true) + private SalesforceBulkQueryOperation operation; + + @GuiWidgetElement( + id = "soql", + order = "200", + type = GuiElementType.MULTI_LINE_TEXT, + multiLineTextHeight = 8, + label = "i18n::SalesforceBulkInput.Soql.Label", + toolTip = "i18n::SalesforceBulkInput.Soql.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "soql") + private String soql; + + @GuiWidgetElement( + id = "timeout", + order = "300", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkInput.Timeout.Label", + toolTip = "i18n::SalesforceBulkInput.Timeout.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "timeout") + private String timeout; + + @GuiWidgetElement( + id = "pollInterval", + order = "400", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkInput.PollInterval.Label", + toolTip = "i18n::SalesforceBulkInput.PollInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "pollInterval") + private String pollInterval; + + @GuiWidgetElement( + id = "maxRows", + order = "500", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkInput.MaxRows.Label", + toolTip = "i18n::SalesforceBulkInput.MaxRows.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "maxRows") + private String maxRows; + + @GuiWidgetElement( + id = "jobIdField", + order = "600", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkInput.JobIdField.Label", + toolTip = "i18n::SalesforceBulkInput.JobIdField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_QUERY, + groupOrder = "200") + @HopMetadataProperty(key = "jobIdField") + private String jobIdField; + + @GuiWidgetElement( + id = "fields", + order = "100", + type = GuiElementType.TABLE, + tableRows = 8, + toolTip = "i18n::SalesforceBulkInput.Fields.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_FIELDS, + groupOrder = "300") + @HopMetadataProperty(key = "field", groupKey = "fields") + private List fields; + + public SalesforceBulkInputMeta() { + setDefault(); + } + + @Override + public void setDefault() { + connection = ""; + operation = SalesforceBulkQueryOperation.QUERY; + soql = "SELECT Id, Name FROM Account"; + timeout = "3600"; + pollInterval = "10"; + maxRows = "0"; + jobIdField = ""; + fields = new ArrayList<>(); + } + + public List hopTypeNames(ILogChannel log, IHopMetadataProvider metadataProvider) { + return List.of("String", "Integer", "Number", "BigNumber", "Date", "Timestamp", "Boolean"); + } + + @Override + public void getFields( + IRowMeta rowMeta, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + rowMeta.clear(); + if (fields != null) { + for (SalesforceBulkInputField field : fields) { + if (field == null || Utils.isEmpty(field.getName())) { + continue; + } + String fieldName = variables.resolve(field.getName()); + String typeName = Utils.isEmpty(field.getType()) ? "String" : field.getType(); + try { + IValueMeta value = + ValueMetaFactory.createValueMeta( + fieldName, ValueMetaFactory.getIdForValueMeta(typeName)); + value.setOrigin(name); + rowMeta.addValueMeta(value); + } catch (org.apache.hop.core.exception.HopPluginException e) { + throw new HopTransformException(e); + } + } + } + if (variables != null && !Utils.isEmpty(variables.resolve(jobIdField))) { + ValueMetaString job = new ValueMetaString(variables.resolve(jobIdField)); + job.setOrigin(name); + rowMeta.addValueMeta(job); + } + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + if (Utils.isEmpty(variables.resolve(connection))) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "SalesforceBulkInput.Check.NoConnection"), + transformMeta)); + } + if (Utils.isEmpty(variables.resolve(soql))) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "SalesforceBulkInput.Check.NoSoql"), + transformMeta)); + } + if (fields == null || fields.isEmpty()) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "SalesforceBulkInput.Check.NoFields"), + transformMeta)); + } + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkJob.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkJob.java new file mode 100644 index 00000000000..d37fdb17f40 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkJob.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import lombok.Getter; + +/** One Bulk API 2.0 ingest or query job. */ +@Getter +public class SalesforceBulkJob { + private final String id; + private final String state; + private final String errorMessage; + private final int recordsProcessed; + private final int recordsFailed; + + public SalesforceBulkJob( + String id, String state, String errorMessage, int recordsProcessed, int recordsFailed) { + this.id = id; + this.state = state; + this.errorMessage = errorMessage; + this.recordsProcessed = recordsProcessed; + this.recordsFailed = recordsFailed; + } + + public boolean isFinished() { + return "JobComplete".equals(state) || "Failed".equals(state) || "Aborted".equals(state); + } + + public boolean isAborted() { + return "Aborted".equals(state); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLineEnding.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLineEnding.java new file mode 100644 index 00000000000..1030684bce5 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLineEnding.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +/** Line ending declared on a Bulk API 2.0 ingest job. The CSV must use the same ending. */ +public enum SalesforceBulkLineEnding implements IEnumHasCodeAndDescription { + LF("LF", "\n", "SalesforceBulkLineEnding.Lf"), + CRLF("CRLF", "\r\n", "SalesforceBulkLineEnding.Crlf"); + + private final String code; + private final String characters; + private final String descriptionKey; + + SalesforceBulkLineEnding(String code, String characters, String descriptionKey) { + this.code = code; + this.characters = characters; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + public String getCharacters() { + return characters; + } + + @Override + public String getDescription() { + return BaseMessages.getString(SalesforceBulkLineEnding.class, descriptionKey); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoad.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoad.java new file mode 100644 index 00000000000..15d7a6bc791 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoad.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** Writes incoming rows with one or more Bulk API 2.0 ingest jobs. */ +public class SalesforceBulkLoad + extends BaseTransform { + private static final Class PKG = SalesforceBulkLoad.class; + + public SalesforceBulkLoad( + TransformMeta transformMeta, + SalesforceBulkLoadMeta meta, + SalesforceBulkLoadData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + if (data.client != null) { + return true; + } + try { + data.client = + SalesforceBulkRuntime.open(this, metadataProvider, getLogChannel(), meta.getConnection()); + return true; + } catch (Exception e) { + logError(e.getMessage(), e); + return false; + } + } + + @Override + public boolean processRow() throws HopException { + Object[] row = getRow(); + if (first) { + first = false; + if (row != null) { + prepare(); + } + } + if (row == null) { + flush(); + setOutputDone(); + return false; + } + List values = new ArrayList<>(data.headers.size()); + for (int i = 0; i < data.streamIndexes.length; i++) { + values.add( + SalesforceBulkValues.format( + data.inputRowMeta.getValueMeta(data.streamIndexes[i]), + row[data.streamIndexes[i]], + data.nullAsNa)); + } + int additional = rowBytes(values); + if (!data.buffer.isEmpty() && data.bufferedBytes + additional > data.maxBytes) { + flush(); + } + data.buffer.add(new SalesforceBulkRowMatcher.Pending(row, values)); + data.bufferedBytes += additional; + return true; + } + + private void prepare() throws HopException { + data.inputRowMeta = getInputRowMeta(); + if (data.inputRowMeta == null) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.NoInput")); + } + data.outputRowMeta = data.inputRowMeta.clone(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + List fields = meta.getFields() == null ? List.of() : meta.getFields(); + if (fields.isEmpty()) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.NoFields")); + } + data.streamIndexes = new int[fields.size()]; + data.headers = new ArrayList<>(); + SalesforceBulkOperation operation = + meta.getOperation() == null ? SalesforceBulkOperation.INSERT : meta.getOperation(); + String externalId = resolve(meta.getExternalIdField()); + int key = -1; + for (int i = 0; i < fields.size(); i++) { + String stream = resolve(fields.get(i).getStreamField()); + String salesforce = resolve(fields.get(i).getSalesforceField()); + if (Utils.isEmpty(stream) || Utils.isEmpty(salesforce)) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.BlankField")); + } + int index = data.inputRowMeta.indexOfValue(stream); + if (index < 0) { + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.MissingField", stream)); + } + data.streamIndexes[i] = index; + data.headers.add(salesforce); + if (operation.isDelete() && "Id".equals(salesforce)) { + key = i; + } + if (operation == SalesforceBulkOperation.UPDATE && "Id".equals(salesforce)) { + key = i; + } + if (operation == SalesforceBulkOperation.UPSERT && salesforce.equals(externalId)) { + key = i; + } + } + if (operation == SalesforceBulkOperation.UPSERT && Utils.isEmpty(externalId)) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.NoExternalId")); + } + if (key < 0 && operation != SalesforceBulkOperation.INSERT) { + throw new HopException( + BaseMessages.getString( + PKG, + operation == SalesforceBulkOperation.UPSERT + ? "SalesforceBulkLoad.Error.ExternalIdNotMapped" + : "SalesforceBulkLoad.Error.IdNotMapped")); + } + data.keyIndexes = + operation == SalesforceBulkOperation.INSERT ? allColumns(fields.size()) : new int[] {key}; + data.nullAsNa = + operation == SalesforceBulkOperation.UPDATE || operation == SalesforceBulkOperation.UPSERT; + data.timeoutMillis = SalesforceBulkRuntime.secondsToMillis(resolve(meta.getTimeout()), 3600); + data.pollMillis = SalesforceBulkRuntime.secondsToMillis(resolve(meta.getPollInterval()), 10); + long maxBytes = longValue(resolve(meta.getMaxBytes())); + data.maxBytes = maxBytes <= 0 ? 104_857_600L : maxBytes; + data.request = + new SalesforceBulkIngestRequest( + resolve(meta.getSalesforceObject()), + operation, + externalId, + meta.getLineEnding() == null ? SalesforceBulkLineEnding.LF : meta.getLineEnding(), + meta.getDelimiter() == null ? SalesforceBulkDelimiter.COMMA : meta.getDelimiter(), + meta.getConcurrency() == null + ? SalesforceBulkConcurrency.PARALLEL + : meta.getConcurrency()); + if (Utils.isEmpty(data.request.getSalesforceObject())) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.NoObject")); + } + data.idIndex = indexOf(meta.getSalesforceIdField()); + data.jobIdIndex = indexOf(meta.getJobIdField()); + data.successIndex = indexOf(meta.getSuccessField()); + data.errorIndex = indexOf(meta.getErrorField()); + SalesforceBulkCsv.Builder headerOnly = + new SalesforceBulkCsv.Builder( + data.headers, data.request.getDelimiter(), data.request.getLineEnding()); + data.bufferedBytes = headerOnly.sizeBytes(); + } + + private int indexOf(String fieldName) { + String resolved = resolve(fieldName); + if (Utils.isEmpty(resolved)) { + return -1; + } + return data.outputRowMeta.indexOfValue(resolved); + } + + private static int[] allColumns(int size) { + int[] indexes = new int[size]; + for (int i = 0; i < size; i++) { + indexes[i] = i; + } + return indexes; + } + + private int rowBytes(List values) { + SalesforceBulkCsv.Builder one = + new SalesforceBulkCsv.Builder( + data.headers, data.request.getDelimiter(), data.request.getLineEnding()); + int header = one.sizeBytes(); + one.add(values); + return one.sizeBytes() - header; + } + + private void flush() throws HopException { + if (data.buffer.isEmpty()) { + return; + } + SalesforceBulkCsv.Builder csv = + new SalesforceBulkCsv.Builder( + data.headers, data.request.getDelimiter(), data.request.getLineEnding()); + for (SalesforceBulkRowMatcher.Pending row : data.buffer) { + csv.add(row.getValues()); + } + SalesforceBulkJob created = data.client.createIngestJob(data.request); + data.openJobId = created.getId(); + data.client.uploadIngestData(created.getId(), csv.toByteArray()); + data.client.completeIngestUpload(created.getId()); + SalesforceBulkJob finished = + SalesforceBulkRuntime.poll( + data.client, + created.getId(), + false, + data.timeoutMillis, + data.pollMillis, + this::isStopped); + data.openJobId = null; + if (finished.isAborted()) { + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.Aborted", created.getId())); + } + logBasic( + BaseMessages.getString( + PKG, + "SalesforceBulkLoad.Log.JobFinished", + created.getId(), + finished.getState(), + String.valueOf(finished.getRecordsProcessed()), + String.valueOf(finished.getRecordsFailed()))); + SalesforceBulkRowMatcher.match( + data.buffer, + data.headers, + data.keyIndexes, + data.client.ingestResults(created.getId(), false), + data.request.getDelimiter(), + false); + SalesforceBulkRowMatcher.match( + data.buffer, + data.headers, + data.keyIndexes, + data.client.ingestResults(created.getId(), true), + data.request.getDelimiter(), + true); + for (SalesforceBulkRowMatcher.Pending row : data.buffer) { + if (!row.isMatched()) { + row.setError( + Utils.isEmpty(finished.getErrorMessage()) + ? BaseMessages.getString(PKG, "SalesforceBulkLoad.Error.NoResult") + : finished.getErrorMessage()); + } + emit(row, created.getId()); + } + data.buffer.clear(); + SalesforceBulkCsv.Builder headerOnly = + new SalesforceBulkCsv.Builder( + data.headers, data.request.getDelimiter(), data.request.getLineEnding()); + data.bufferedBytes = headerOnly.sizeBytes(); + } + + private void emit(SalesforceBulkRowMatcher.Pending row, String jobId) throws HopException { + boolean success = row.isMatched() && Utils.isEmpty(row.getError()); + if (!success && getTransformMeta().isDoingErrorHandling()) { + putError( + data.inputRowMeta, + row.getInputRow(), + 1, + Utils.isEmpty(row.getError()) ? "failed" : row.getError(), + null, + "SalesforceBulkLoad001"); + return; + } + Object[] output = RowDataUtil.resizeArray(row.getInputRow(), data.outputRowMeta.size()); + if (data.idIndex >= 0) { + output[data.idIndex] = row.getSalesforceId(); + } + if (data.jobIdIndex >= 0) { + output[data.jobIdIndex] = jobId; + } + if (data.successIndex >= 0) { + output[data.successIndex] = success; + } + if (data.errorIndex >= 0) { + output[data.errorIndex] = success ? null : row.getError(); + } + putRow(data.outputRowMeta, output); + } + + private static long longValue(String text) { + if (Utils.isEmpty(text)) { + return -1; + } + try { + return Long.parseLong(text.trim()); + } catch (NumberFormatException e) { + return -1; + } + } + + @Override + public void dispose() { + if (data.openJobId != null) { + SalesforceBulkRuntime.abort(data.client, data.openJobId, false); + data.openJobId = null; + } + if (data.client != null) { + data.client.close(); + data.client = null; + } + super.dispose(); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadData.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadData.java new file mode 100644 index 00000000000..4fc4b76ae08 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadData.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; +import org.apache.hop.pipeline.transforms.salesforce.SalesforceConnection; + +@SuppressWarnings("java:S1104") +public class SalesforceBulkLoadData extends BaseTransformData implements ITransformData { + public SalesforceConnection connection; + public SalesforceBulkClient client; + public IRowMeta inputRowMeta; + public IRowMeta outputRowMeta; + public int[] streamIndexes; + public List headers = new ArrayList<>(); + public int[] keyIndexes = new int[0]; + public boolean nullAsNa; + public long timeoutMillis = 3_600_000L; + public long pollMillis = 10_000L; + public long maxBytes = 104_857_600L; + public SalesforceBulkIngestRequest request; + public int idIndex = -1; + public int jobIdIndex = -1; + public int successIndex = -1; + public int errorIndex = -1; + public final List buffer = new ArrayList<>(); + public String openJobId; + public long bufferedBytes; +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadDialog.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadDialog.java new file mode 100644 index 00000000000..f76eff619cf --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadDialog.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.widgets.Shell; + +public class SalesforceBulkLoadDialog extends BaseTransformDialog { + private static final Class PKG = SalesforceBulkLoadMeta.class; + + private final SalesforceBulkLoadMeta input; + private GuiCompositeWidgets widgets; + + public SalesforceBulkLoadDialog( + Shell parent, + IVariables variables, + SalesforceBulkLoadMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + createShell(BaseMessages.getString(PKG, "SalesforceBulkLoad.Name")); + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + widgets = + GuiCompositeWidgets.addScrolledComposite( + shell, + variables, + wTransformName, + wOk, + SalesforceBulkLoadMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, + input); + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + private void cancel() { + transformName = null; + dispose(); + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + widgets.getWidgetsContents(input, SalesforceBulkLoadMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + transformName = wTransformName.getText(); + dispose(); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadMeta.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadMeta.java new file mode 100644 index 00000000000..723e4503698 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadMeta.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.salesforce.SalesforceConnection; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@GuiPlugin +@Transform( + id = "SalesforceBulkLoad", + name = "i18n::SalesforceBulkLoad.Name", + description = "i18n::SalesforceBulkLoad.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Output", + keywords = "i18n::SalesforceBulkLoad.keyword", + image = "salesforcebulkload.svg", + documentationUrl = "/pipeline/transforms/salesforcebulkload.html", + excludedEngines = {"Beam*", "SparkPipelineEngine"}) +public class SalesforceBulkLoadMeta + extends BaseTransformMeta { + private static final Class PKG = SalesforceBulkLoadMeta.class; + + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "SalesforceBulkLoad.Dialog"; + private static final String GROUP_CONNECTION = "i18n::SalesforceBulkLoad.Group.Connection"; + private static final String GROUP_OPERATION = "i18n::SalesforceBulkLoad.Group.Operation"; + private static final String GROUP_FIELDS = "i18n::SalesforceBulkLoad.Group.Fields"; + + @GuiWidgetElement( + id = "connection", + order = "100", + type = GuiElementType.METADATA, + metadata = SalesforceConnection.class, + label = "i18n::SalesforceBulkLoad.Connection.Label", + toolTip = "i18n::SalesforceBulkLoad.Connection.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_CONNECTION, + groupOrder = "100") + @HopMetadataProperty(key = "connection") + private String connection; + + @GuiWidgetElement( + id = "salesforceObject", + order = "100", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.Object.Label", + toolTip = "i18n::SalesforceBulkLoad.Object.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "object") + private String salesforceObject; + + @GuiWidgetElement( + id = "operation", + order = "200", + type = GuiElementType.COMBO, + label = "i18n::SalesforceBulkLoad.Operation.Label", + toolTip = "i18n::SalesforceBulkLoad.Operation.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "operation", storeWithCode = true) + private SalesforceBulkOperation operation; + + @GuiWidgetElement( + id = "externalIdField", + order = "300", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.ExternalId.Label", + toolTip = "i18n::SalesforceBulkLoad.ExternalId.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "externalIdField") + private String externalIdField; + + @GuiWidgetElement( + id = "concurrency", + order = "400", + type = GuiElementType.COMBO, + label = "i18n::SalesforceBulkLoad.Concurrency.Label", + toolTip = "i18n::SalesforceBulkLoad.Concurrency.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "concurrency", storeWithCode = true) + private SalesforceBulkConcurrency concurrency; + + @GuiWidgetElement( + id = "lineEnding", + order = "500", + type = GuiElementType.COMBO, + label = "i18n::SalesforceBulkLoad.LineEnding.Label", + toolTip = "i18n::SalesforceBulkLoad.LineEnding.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "lineEnding", storeWithCode = true) + private SalesforceBulkLineEnding lineEnding; + + @GuiWidgetElement( + id = "delimiter", + order = "600", + type = GuiElementType.COMBO, + label = "i18n::SalesforceBulkLoad.Delimiter.Label", + toolTip = "i18n::SalesforceBulkLoad.Delimiter.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "delimiter", storeWithCode = true) + private SalesforceBulkDelimiter delimiter; + + @GuiWidgetElement( + id = "timeout", + order = "700", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.Timeout.Label", + toolTip = "i18n::SalesforceBulkLoad.Timeout.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "timeout") + private String timeout; + + @GuiWidgetElement( + id = "pollInterval", + order = "800", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.PollInterval.Label", + toolTip = "i18n::SalesforceBulkLoad.PollInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "pollInterval") + private String pollInterval; + + @GuiWidgetElement( + id = "maxBytes", + order = "900", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.MaxBytes.Label", + toolTip = "i18n::SalesforceBulkLoad.MaxBytes.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "maxBytes") + private String maxBytes; + + @GuiWidgetElement( + id = "salesforceIdField", + order = "1000", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.IdField.Label", + toolTip = "i18n::SalesforceBulkLoad.IdField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "salesforceIdField") + private String salesforceIdField; + + @GuiWidgetElement( + id = "jobIdField", + order = "1100", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.JobIdField.Label", + toolTip = "i18n::SalesforceBulkLoad.JobIdField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "jobIdField") + private String jobIdField; + + @GuiWidgetElement( + id = "successField", + order = "1200", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.SuccessField.Label", + toolTip = "i18n::SalesforceBulkLoad.SuccessField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "successField") + private String successField; + + @GuiWidgetElement( + id = "errorField", + order = "1300", + type = GuiElementType.TEXT, + label = "i18n::SalesforceBulkLoad.ErrorField.Label", + toolTip = "i18n::SalesforceBulkLoad.ErrorField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_OPERATION, + groupOrder = "200") + @HopMetadataProperty(key = "errorField") + private String errorField; + + @GuiWidgetElement( + id = "fields", + order = "100", + type = GuiElementType.TABLE, + tableRows = 8, + toolTip = "i18n::SalesforceBulkLoad.Fields.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = GROUP_FIELDS, + groupOrder = "300") + @HopMetadataProperty(key = "field", groupKey = "fields") + private List fields; + + public SalesforceBulkLoadMeta() { + setDefault(); + } + + @Override + public void setDefault() { + connection = ""; + salesforceObject = "Account"; + operation = SalesforceBulkOperation.INSERT; + externalIdField = ""; + concurrency = SalesforceBulkConcurrency.PARALLEL; + lineEnding = SalesforceBulkLineEnding.LF; + delimiter = SalesforceBulkDelimiter.COMMA; + timeout = "3600"; + pollInterval = "10"; + maxBytes = "104857600"; + salesforceIdField = "SalesforceId"; + jobIdField = "SalesforceJobId"; + successField = "SalesforceSuccess"; + errorField = "SalesforceError"; + fields = new ArrayList<>(); + } + + @Override + public void getFields( + IRowMeta rowMeta, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) { + add(rowMeta, variables, name, salesforceIdField, false); + add(rowMeta, variables, name, jobIdField, false); + add(rowMeta, variables, name, successField, true); + add(rowMeta, variables, name, errorField, false); + } + + private static void add( + IRowMeta rowMeta, IVariables variables, String origin, String fieldName, boolean bool) { + if (variables == null || Utils.isEmpty(variables.resolve(fieldName))) { + return; + } + String resolved = variables.resolve(fieldName); + if (bool) { + ValueMetaBoolean value = new ValueMetaBoolean(resolved); + value.setOrigin(origin); + rowMeta.addValueMeta(value); + } else { + ValueMetaString value = new ValueMetaString(resolved); + value.setOrigin(origin); + rowMeta.addValueMeta(value); + } + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + if (input == null || input.length == 0) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.NoInput"); + } + if (Utils.isEmpty(variables.resolve(connection))) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.NoConnection"); + } + if (Utils.isEmpty(variables.resolve(salesforceObject))) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.NoObject"); + } + if (fields == null || fields.isEmpty()) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.NoFields"); + } else if (prev != null) { + for (SalesforceBulkField field : fields) { + if (field == null || Utils.isEmpty(field.getStreamField())) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.BlankField"); + continue; + } + if (prev.indexOfValue(variables.resolve(field.getStreamField())) < 0) { + error( + remarks, + transformMeta, + "SalesforceBulkLoad.Check.MissingField", + field.getStreamField()); + } + } + } + if (operation == SalesforceBulkOperation.UPSERT + && Utils.isEmpty(variables.resolve(externalIdField))) { + error(remarks, transformMeta, "SalesforceBulkLoad.Check.NoExternalId"); + } + } + + private void error( + List remarks, TransformMeta transformMeta, String key, String... args) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, key, args), transformMeta)); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkOperation.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkOperation.java new file mode 100644 index 00000000000..9dda8481a96 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkOperation.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +/** Bulk API 2.0 ingest operation. {@link #getCode()} is the value Salesforce stores. */ +public enum SalesforceBulkOperation implements IEnumHasCodeAndDescription { + INSERT("insert", "SalesforceBulkOperation.Insert"), + UPDATE("update", "SalesforceBulkOperation.Update"), + UPSERT("upsert", "SalesforceBulkOperation.Upsert"), + DELETE("delete", "SalesforceBulkOperation.Delete"), + HARD_DELETE("hardDelete", "SalesforceBulkOperation.HardDelete"); + + private final String code; + private final String descriptionKey; + + SalesforceBulkOperation(String code, String descriptionKey) { + this.code = code; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return BaseMessages.getString(SalesforceBulkOperation.class, descriptionKey); + } + + public boolean isDelete() { + return this == DELETE || this == HARD_DELETE; + } + + public static SalesforceBulkOperation lookup(String descriptionOrCode) { + SalesforceBulkOperation byDescription = + IEnumHasCodeAndDescription.lookupDescription( + SalesforceBulkOperation.class, descriptionOrCode, null); + if (byDescription != null) { + return byDescription; + } + if (descriptionOrCode == null) { + return INSERT; + } + for (SalesforceBulkOperation operation : values()) { + if (operation.code.equalsIgnoreCase(descriptionOrCode) + || operation.name().equalsIgnoreCase(descriptionOrCode)) { + return operation; + } + } + return INSERT; + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryOperation.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryOperation.java new file mode 100644 index 00000000000..c9ce25703b3 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryOperation.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +/** Bulk API 2.0 query operation. {@code queryAll} includes deleted and archived rows. */ +public enum SalesforceBulkQueryOperation implements IEnumHasCodeAndDescription { + QUERY("query", "SalesforceBulkQueryOperation.Query"), + QUERY_ALL("queryAll", "SalesforceBulkQueryOperation.QueryAll"); + + private final String code; + private final String descriptionKey; + + SalesforceBulkQueryOperation(String code, String descriptionKey) { + this.code = code; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return BaseMessages.getString(SalesforceBulkQueryOperation.class, descriptionKey); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryPage.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryPage.java new file mode 100644 index 00000000000..6a07dc1e791 --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkQueryPage.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import lombok.Getter; + +/** One page of a Bulk API 2.0 query result. A null or literal {@code null} locator ends the job. */ +@Getter +public class SalesforceBulkQueryPage { + private final String csv; + private final String locator; + + public SalesforceBulkQueryPage(String csv, String locator) { + this.csv = csv == null ? "" : csv; + this.locator = locator; + } + + public boolean hasMore() { + return locator != null && !locator.isBlank() && !"null".equalsIgnoreCase(locator); + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcher.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcher.java new file mode 100644 index 00000000000..e5b6abe153f --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcher.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.exception.HopException; + +/** + * Pairs Bulk API result rows with the rows that were uploaded. Result files are not in input order. + * Rows that share a key are paired in input order. + */ +public final class SalesforceBulkRowMatcher { + private SalesforceBulkRowMatcher() {} + + /** One uploaded row and the result Salesforce returned for it. */ + @Getter + public static final class Pending { + private final Object[] inputRow; + private final List values; + @Setter private boolean matched; + @Setter private String salesforceId; + @Setter private String error; + + public Pending(Object[] inputRow, List values) { + this.inputRow = inputRow; + this.values = values; + } + } + + public static String key(List values, int[] keyIndexes) { + StringBuilder key = new StringBuilder(); + for (int i = 0; i < keyIndexes.length; i++) { + if (i > 0) { + key.append('\u0001'); + } + String value = values.get(keyIndexes[i]); + key.append(value == null ? "" : value); + } + return key.toString(); + } + + public static void match( + List pending, + List sentHeaders, + int[] keyIndexes, + String resultsCsv, + SalesforceBulkDelimiter delimiter, + boolean failed) + throws HopException { + if (resultsCsv == null || resultsCsv.isBlank() || pending.isEmpty()) { + return; + } + List> table = SalesforceBulkCsv.parse(resultsCsv, delimiter); + if (table.size() < 2) { + return; + } + List resultHeaders = table.get(0); + Map index = new HashMap<>(); + for (int i = 0; i < resultHeaders.size(); i++) { + index.put(resultHeaders.get(i), i); + } + int idIndex = index.getOrDefault(SalesforceBulkCsv.SUCCESS_ID, -1); + int errorIndex = index.getOrDefault(SalesforceBulkCsv.FAILURE_ERROR, -1); + for (int column : keyIndexes) { + if (!index.containsKey(sentHeaders.get(column))) { + throw new HopException("Bulk API results have no column " + sentHeaders.get(column)); + } + } + Map> waiting = new HashMap<>(); + for (Pending row : pending) { + if (row.isMatched()) { + continue; + } + waiting + .computeIfAbsent(key(row.getValues(), keyIndexes), ignored -> new ArrayDeque<>()) + .add(row); + } + for (int r = 1; r < table.size(); r++) { + List result = table.get(r); + List aligned = new ArrayList<>(sentHeaders.size()); + for (int c = 0; c < sentHeaders.size(); c++) { + int at = index.getOrDefault(sentHeaders.get(c), -1); + aligned.add(at >= 0 && at < result.size() ? result.get(at) : ""); + } + ArrayDeque queue = waiting.get(key(aligned, keyIndexes)); + if (queue == null || queue.isEmpty()) { + continue; + } + Pending row = queue.removeFirst(); + row.setMatched(true); + if (idIndex >= 0 && idIndex < result.size()) { + row.setSalesforceId(result.get(idIndex)); + } + if (failed && errorIndex >= 0 && errorIndex < result.size()) { + row.setError(result.get(errorIndex)); + } + } + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRuntime.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRuntime.java new file mode 100644 index 00000000000..c28cf1b5ccb --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRuntime.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.util.function.BooleanSupplier; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.api.IHopMetadataSerializer; +import org.apache.hop.pipeline.transforms.salesforce.SalesforceConnection; + +/** Opens a Bulk API client from a Salesforce connection and polls a job to a final state. */ +public final class SalesforceBulkRuntime { + private static final Class PKG = SalesforceBulkRuntime.class; + + private SalesforceBulkRuntime() {} + + public static SalesforceBulkClient open( + IVariables variables, + IHopMetadataProvider metadataProvider, + ILogChannel log, + String connectionName) + throws HopException { + String name = variables.resolve(connectionName); + if (Utils.isEmpty(name)) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulk.Error.NoConnection")); + } + IHopMetadataSerializer serializer = + metadataProvider.getSerializer( + org.apache.hop.metadata.salesforce.SalesforceConnection.class); + org.apache.hop.metadata.salesforce.SalesforceConnection metadata = serializer.load(name); + if (metadata == null) { + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulk.Error.MissingConnection", name)); + } + SalesforceConnection connection = metadata.createConnection(variables, log); + boolean tokenReady = + !Utils.isEmpty(connection.getOauthAccessToken()) + && !Utils.isEmpty(connection.getOauthInstanceUrl()); + if (!tokenReady) { + connection.connect(); + } + return new SalesforceBulkHttpClient(SalesforceBulkSession.from(connection)); + } + + public static long secondsToMillis(String resolved, long defaultSeconds) { + int seconds = Const.toInt(resolved, (int) defaultSeconds); + if (seconds <= 0) { + seconds = (int) defaultSeconds; + } + return seconds * 1000L; + } + + public static SalesforceBulkJob poll( + SalesforceBulkClient client, + String jobId, + boolean query, + long timeoutMillis, + long pollMillis, + BooleanSupplier stopped) + throws HopException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (true) { + if (stopped.getAsBoolean()) { + abort(client, jobId, query); + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulk.Error.Stopped", jobId)); + } + SalesforceBulkJob job = query ? client.getQueryJob(jobId) : client.getIngestJob(jobId); + if (job.isFinished()) { + return job; + } + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + abort(client, jobId, query); + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulk.Error.Timeout", jobId, job.getState())); + } + try { + Thread.sleep(Math.min(pollMillis, remaining)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(client, jobId, query); + throw new HopException( + BaseMessages.getString(PKG, "SalesforceBulk.Error.Stopped", jobId), e); + } + } + } + + public static void abort(SalesforceBulkClient client, String jobId, boolean query) { + if (client == null || Utils.isEmpty(jobId)) { + return; + } + try { + if (query) { + client.abortQueryJob(jobId); + } else { + client.abortIngestJob(jobId); + } + } catch (Exception ignored) { + // The pipeline is already leaving the job. + } + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSession.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSession.java new file mode 100644 index 00000000000..30c19f6cffe --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSession.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.net.URI; +import lombok.Getter; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.transforms.salesforce.SalesforceConnection; + +/** Bearer token and instance root for Bulk API 2.0. */ +@Getter +public class SalesforceBulkSession { + private static final Class PKG = SalesforceBulkSession.class; + + private final String instanceUrl; + private final String accessToken; + private final String apiVersion; + + public SalesforceBulkSession(String instanceUrl, String accessToken, String apiVersion) { + this.instanceUrl = instanceUrl; + this.accessToken = accessToken; + this.apiVersion = apiVersion; + } + + /** + * Read the session from a connection that already has an OAuth token, or that has logged in with + * a username and password. The SOAP session id is the bearer token. The REST host is the host of + * the SOAP server URL. + */ + public static SalesforceBulkSession from(SalesforceConnection connection) throws HopException { + String token = connection.getOauthAccessToken(); + String instance = connection.getOauthInstanceUrl(); + if (Utils.isEmpty(token) && connection.getLoginResult() != null) { + token = connection.getLoginResult().getSessionId(); + } + if (Utils.isEmpty(instance) && connection.getLoginResult() != null) { + instance = connection.getLoginResult().getServerUrl(); + } + instance = instanceRoot(instance); + if (Utils.isEmpty(token) || Utils.isEmpty(instance)) { + throw new HopException(BaseMessages.getString(PKG, "SalesforceBulk.Error.MissingSession")); + } + return new SalesforceBulkSession(instance, token, apiVersion(connection.getApiVersion())); + } + + /** Drop a SOAP or REST path so the result is {@code scheme://host[:port]}. */ + public static String instanceRoot(String url) { + if (Utils.isEmpty(url)) { + return null; + } + String trimmed = url.trim(); + int services = trimmed.indexOf("/services/"); + if (services > 0) { + trimmed = trimmed.substring(0, services); + } + while (trimmed.endsWith("/")) { + trimmed = trimmed.substring(0, trimmed.length() - 1); + } + try { + URI uri = URI.create(trimmed); + if (uri.getScheme() != null && uri.getRawAuthority() != null) { + return uri.getScheme() + "://" + uri.getRawAuthority(); + } + } catch (IllegalArgumentException ignored) { + // Fall through to the trimmed text. + } + return trimmed; + } + + static String apiVersion(String version) { + if (Utils.isEmpty(version)) { + return "64.0"; + } + String trimmed = version.trim(); + if (trimmed.startsWith("v") || trimmed.startsWith("V")) { + trimmed = trimmed.substring(1); + } + return trimmed; + } +} diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValues.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValues.java new file mode 100644 index 00000000000..5e4a054876f --- /dev/null +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValues.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import java.math.BigDecimal; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; + +/** Converts Hop values to Bulk API CSV text and CSV text back to Hop values. */ +public final class SalesforceBulkValues { + private static final String[] DATE_PATTERNS = { + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ss.SSSX", + "yyyy-MM-dd'T'HH:mm:ssX", + "yyyy-MM-dd" + }; + + private SalesforceBulkValues() {} + + public static String format(IValueMeta valueMeta, Object value, boolean nullAsNa) + throws HopValueException { + if (valueMeta.isNull(value)) { + return nullAsNa ? SalesforceBulkCsv.NULL_TOKEN : ""; + } + return switch (valueMeta.getType()) { + case IValueMeta.TYPE_BOOLEAN -> + Boolean.TRUE.equals(valueMeta.getBoolean(value)) ? "true" : "false"; + case IValueMeta.TYPE_INTEGER -> Long.toString(valueMeta.getInteger(value)); + case IValueMeta.TYPE_NUMBER -> + BigDecimal.valueOf(valueMeta.getNumber(value)).stripTrailingZeros().toPlainString(); + case IValueMeta.TYPE_BIGNUMBER -> valueMeta.getBigNumber(value).toPlainString(); + case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> formatDate(valueMeta.getDate(value)); + default -> valueMeta.getString(value); + }; + } + + public static Object parse(IValueMeta target, String text) throws HopException { + if (text == null || text.isEmpty()) { + return null; + } + try { + return switch (target.getType()) { + case IValueMeta.TYPE_BOOLEAN -> "true".equalsIgnoreCase(text) || "1".equals(text); + case IValueMeta.TYPE_INTEGER -> Long.valueOf(text); + case IValueMeta.TYPE_NUMBER -> Double.valueOf(text); + case IValueMeta.TYPE_BIGNUMBER -> new BigDecimal(text); + case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> parseDate(text); + default -> text; + }; + } catch (RuntimeException e) { + throw new HopException("Cannot read '" + text + "' as " + target.getTypeDesc(), e); + } + } + + static String formatDate(Date date) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + return format.format(date); + } + + static Date parseDate(String text) { + for (String pattern : DATE_PATTERNS) { + SimpleDateFormat format = new SimpleDateFormat(pattern); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + format.setLenient(false); + ParsePosition position = new ParsePosition(0); + Date parsed = format.parse(text, position); + if (parsed != null && position.getIndex() == text.length()) { + return parsed; + } + } + throw new IllegalArgumentException(text); + } +} diff --git a/plugins/tech/salesforce/src/main/resources/org/apache/hop/pipeline/transforms/salesforcebulk/messages/messages_en_US.properties b/plugins/tech/salesforce/src/main/resources/org/apache/hop/pipeline/transforms/salesforcebulk/messages/messages_en_US.properties new file mode 100644 index 00000000000..6f09e7390b8 --- /dev/null +++ b/plugins/tech/salesforce/src/main/resources/org/apache/hop/pipeline/transforms/salesforcebulk/messages/messages_en_US.properties @@ -0,0 +1,129 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SalesforceBulkLoad.Name=Salesforce bulk load (experimental) +SalesforceBulkLoad.Description=Insert, update, upsert, or delete Salesforce records with Bulk API 2.0. Experimental. +SalesforceBulkLoad.keyword=salesforce,bulk,load,insert,update,upsert,delete +SalesforceBulkLoad.Group.Connection=Connection +SalesforceBulkLoad.Group.Operation=Operation +SalesforceBulkLoad.Group.Fields=Fields +SalesforceBulkLoad.Connection.Label=Salesforce connection +SalesforceBulkLoad.Connection.Tooltip=Connection that supplies the access token and instance URL. +SalesforceBulkLoad.Object.Label=Salesforce object +SalesforceBulkLoad.Object.Tooltip=API name of the object, for example Account. +SalesforceBulkLoad.Operation.Label=Operation +SalesforceBulkLoad.Operation.Tooltip=Bulk API 2.0 ingest operation. One object per job. +SalesforceBulkLoad.ExternalId.Label=External id field +SalesforceBulkLoad.ExternalId.Tooltip=Salesforce field used to match rows for an upsert. +SalesforceBulkLoad.Concurrency.Label=Concurrency +SalesforceBulkLoad.Concurrency.Tooltip=Serial processing avoids row-lock errors when many rows share a parent. +SalesforceBulkLoad.LineEnding.Label=Line ending +SalesforceBulkLoad.LineEnding.Tooltip=Line ending written into the CSV. It must match the job. +SalesforceBulkLoad.Delimiter.Label=Column delimiter +SalesforceBulkLoad.Delimiter.Tooltip=Delimiter written into the CSV. It must match the job. +SalesforceBulkLoad.Timeout.Label=Timeout (seconds) +SalesforceBulkLoad.Timeout.Tooltip=How long to wait for a job to finish. The default is 3600. +SalesforceBulkLoad.PollInterval.Label=Poll interval (seconds) +SalesforceBulkLoad.PollInterval.Tooltip=How often to ask Salesforce for the job state. The default is 10. +SalesforceBulkLoad.MaxBytes.Label=Maximum bytes per job +SalesforceBulkLoad.MaxBytes.Tooltip=Start another job when the CSV reaches this size. Stay under the 150 MB Bulk API limit. The default is 104857600. +SalesforceBulkLoad.IdField.Label=Salesforce id field +SalesforceBulkLoad.IdField.Tooltip=Output field that receives sf__Id. Leave it empty to omit the field. +SalesforceBulkLoad.JobIdField.Label=Job id field +SalesforceBulkLoad.JobIdField.Tooltip=Output field that receives the Bulk API job id. Leave it empty to omit the field. +SalesforceBulkLoad.SuccessField.Label=Success field +SalesforceBulkLoad.SuccessField.Tooltip=Output field set to true when Salesforce accepted the row. +SalesforceBulkLoad.ErrorField.Label=Error field +SalesforceBulkLoad.ErrorField.Tooltip=Output field that receives sf__Error when the row is not sent to an error hop. +SalesforceBulkLoad.Fields.Tooltip=Stream field and the Salesforce field API name. Header case must match the API name. +SalesforceBulkLoad.Check.NoInput=This transform needs input rows. +SalesforceBulkLoad.Check.NoConnection=A Salesforce connection is required. +SalesforceBulkLoad.Check.NoObject=A Salesforce object is required. +SalesforceBulkLoad.Check.NoFields=At least one field mapping is required. +SalesforceBulkLoad.Check.BlankField=A field mapping is missing a stream field or a Salesforce field. +SalesforceBulkLoad.Check.MissingField=Field [{0}] is not in the input stream. +SalesforceBulkLoad.Check.NoExternalId=An upsert needs an external id field. +SalesforceBulkLoad.Error.NoInput=Salesforce bulk load received no input row metadata. +SalesforceBulkLoad.Error.NoFields=Salesforce bulk load has no field mappings. +SalesforceBulkLoad.Error.BlankField=A Salesforce bulk load field mapping is incomplete. +SalesforceBulkLoad.Error.MissingField=Field [{0}] could not be found in the input stream. +SalesforceBulkLoad.Error.NoExternalId=An upsert needs an external id field. +SalesforceBulkLoad.Error.ExternalIdNotMapped=The external id field is not in the field mappings. +SalesforceBulkLoad.Error.IdNotMapped=Update and delete need a field mapped to Id. +SalesforceBulkLoad.Error.NoObject=A Salesforce object is required. +SalesforceBulkLoad.Error.Aborted=Salesforce bulk job {0} was aborted. +SalesforceBulkLoad.Error.NoResult=Salesforce returned no result for this row. +SalesforceBulkLoad.Log.JobFinished=Salesforce bulk job {0} finished with state {1}. Processed {2}, failed {3}. + +SalesforceBulkInput.Name=Salesforce bulk input (experimental) +SalesforceBulkInput.Description=Read a large SOQL query with Bulk API 2.0. Experimental. +SalesforceBulkInput.keyword=salesforce,bulk,input,query,soql +SalesforceBulkInput.Group.Connection=Connection +SalesforceBulkInput.Group.Query=Query +SalesforceBulkInput.Group.Fields=Fields +SalesforceBulkInput.Connection.Label=Salesforce connection +SalesforceBulkInput.Connection.Tooltip=Connection that supplies the access token and instance URL. +SalesforceBulkInput.Operation.Label=Operation +SalesforceBulkInput.Operation.Tooltip=queryAll includes deleted and archived rows. +SalesforceBulkInput.Soql.Label=SOQL +SalesforceBulkInput.Soql.Tooltip=SOQL statement. Bulk query does not support aggregates, GROUP BY, OFFSET, or TYPEOF. +SalesforceBulkInput.Timeout.Label=Timeout (seconds) +SalesforceBulkInput.Timeout.Tooltip=How long to wait for the query job to finish. The default is 3600. +SalesforceBulkInput.PollInterval.Label=Poll interval (seconds) +SalesforceBulkInput.PollInterval.Tooltip=How often to ask Salesforce for the job state. The default is 10. +SalesforceBulkInput.MaxRows.Label=Maximum rows +SalesforceBulkInput.MaxRows.Tooltip=Stop after this many result rows. 0 reads every row Salesforce returns. +SalesforceBulkInput.JobIdField.Label=Job id field +SalesforceBulkInput.JobIdField.Tooltip=Optional output field that receives the Bulk API job id. +SalesforceBulkInput.Fields.Tooltip=Output columns. The name must match a column in the query result. +SalesforceBulkInput.Check.NoConnection=A Salesforce connection is required. +SalesforceBulkInput.Check.NoSoql=A SOQL statement is required. +SalesforceBulkInput.Check.NoFields=At least one output field is required. +SalesforceBulkInput.Error.NoFields=Salesforce bulk input has no output fields. +SalesforceBulkInput.Error.NoSoql=A SOQL statement is required. +SalesforceBulkInput.Error.QueryFailed=Salesforce bulk query {0} failed: {1} +SalesforceBulkInput.Error.MissingColumn=Query results have no column {0}. +SalesforceBulkInput.Log.JobFinished=Salesforce bulk query {0} finished. + +SalesforceBulkField.Stream.Label=Stream field +SalesforceBulkField.Salesforce.Label=Salesforce field +SalesforceBulkInputField.Name.Label=Column +SalesforceBulkInputField.Type.Label=Type + +SalesforceBulkOperation.Insert=Insert +SalesforceBulkOperation.Update=Update +SalesforceBulkOperation.Upsert=Upsert +SalesforceBulkOperation.Delete=Delete +SalesforceBulkOperation.HardDelete=Hard delete +SalesforceBulkQueryOperation.Query=Query +SalesforceBulkQueryOperation.QueryAll=Query all +SalesforceBulkConcurrency.Parallel=Parallel +SalesforceBulkConcurrency.Serial=Serial +SalesforceBulkLineEnding.Lf=LF +SalesforceBulkLineEnding.Crlf=CRLF +SalesforceBulkDelimiter.Comma=Comma +SalesforceBulkDelimiter.Semicolon=Semicolon +SalesforceBulkDelimiter.Tab=Tab +SalesforceBulkDelimiter.Pipe=Pipe +SalesforceBulkDelimiter.Caret=Caret +SalesforceBulkDelimiter.Backquote=Backquote + +SalesforceBulk.Error.MissingSession=The Salesforce connection has no access token or instance URL. +SalesforceBulk.Error.NoConnection=A Salesforce connection is required. +SalesforceBulk.Error.MissingConnection=Salesforce connection [{0}] was not found. +SalesforceBulk.Error.Stopped=The pipeline stopped while waiting for Salesforce bulk job {0}. +SalesforceBulk.Error.Timeout=Timed out waiting for Salesforce bulk job {0} in state {1}. diff --git a/plugins/tech/salesforce/src/main/resources/salesforcebulkinput.svg b/plugins/tech/salesforce/src/main/resources/salesforcebulkinput.svg new file mode 100644 index 00000000000..bfbe23a063d --- /dev/null +++ b/plugins/tech/salesforce/src/main/resources/salesforcebulkinput.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/tech/salesforce/src/main/resources/salesforcebulkload.svg b/plugins/tech/salesforce/src/main/resources/salesforcebulkload.svg new file mode 100644 index 00000000000..bfbe23a063d --- /dev/null +++ b/plugins/tech/salesforce/src/main/resources/salesforcebulkload.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-input.hpl b/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-input.hpl new file mode 100644 index 00000000000..d54754445cd --- /dev/null +++ b/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-input.hpl @@ -0,0 +1,99 @@ + + + + + salesforce-bulk-input + Y + + + + Normal + + + N + 1000 + 100 + - + 2026/10/04 09:00:00.000 + - + 2026/10/04 09:00:00.000 + + + + Experimental Bulk API 2.0 query. Requires a Salesforce connection named in the pipeline. + 128 + 16 + 400 + 27 + Noto Sans + 11 + N + N + 14 + 58 + 90 + 201 + 232 + 251 + 14 + 58 + 90 + + + + + + bulk input + SalesforceBulkInput + + Y + + 1 + + none + + + + query + SELECT Id, Name FROM Account + 3600 + 10 + 0 + + + + Id + String + + + Name + String + + + + + 400 + 160 + + + + + + diff --git a/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-load.hpl b/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-load.hpl new file mode 100644 index 00000000000..adae4b19d29 --- /dev/null +++ b/plugins/tech/salesforce/src/main/samples/transforms/salesforce-bulk-load.hpl @@ -0,0 +1,142 @@ + + + + + salesforce-bulk-load + Y + + + + Normal + + + N + 1000 + 100 + - + 2026/10/04 09:00:00.000 + - + 2026/10/04 09:00:00.000 + + + + Experimental Bulk API 2.0 load. Requires a Salesforce connection named in the pipeline. + 128 + 16 + 400 + 27 + Noto Sans + 11 + N + N + 14 + 58 + 90 + 201 + 232 + 251 + 14 + 58 + 90 + + + + + rows + bulk load + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + Name + String + + -1 + -1 + N + + + + + Alice + + + Bob + + + + + 128 + 160 + + + + bulk load + SalesforceBulkLoad + + Y + + 1 + + none + + + + Account + insert + + Parallel + LF + COMMA + 3600 + 10 + 104857600 + SalesforceId + SalesforceJobId + SalesforceSuccess + SalesforceError + + + Name + Name + + + + + 400 + 160 + + + + + + diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsvTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsvTest.java new file mode 100644 index 00000000000..712b57230b5 --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkCsvTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.junit.jupiter.api.Test; + +class SalesforceBulkCsvTest { + + @Test + void quotesCommasQuotesAndSpaces() { + assertEquals("Alice", SalesforceBulkCsv.encode("Alice", ",")); + assertEquals("\"Director, Marketing\"", SalesforceBulkCsv.encode("Director, Marketing", ",")); + assertEquals("\"Say \"\"hi\"\"\"", SalesforceBulkCsv.encode("Say \"hi\"", ",")); + assertEquals("\" Alice\"", SalesforceBulkCsv.encode(" Alice", ",")); + assertEquals("#N/A", SalesforceBulkCsv.encode("#N/A", ",")); + } + + @Test + void roundTripsQuotedFieldsAndSkipsTheTrailingNewline() throws Exception { + SalesforceBulkCsv.Builder builder = + new SalesforceBulkCsv.Builder( + List.of("Name", "City"), SalesforceBulkDelimiter.COMMA, SalesforceBulkLineEnding.LF); + builder.add(List.of("Alice", "Director, Marketing")); + builder.add(List.of("Bob", "")); + String csv = new String(builder.toByteArray(), StandardCharsets.UTF_8); + assertTrue(csv.endsWith("\n")); + List> rows = SalesforceBulkCsv.parse(csv); + assertEquals(List.of("Name", "City"), rows.get(0)); + assertEquals(List.of("Alice", "Director, Marketing"), rows.get(1)); + assertEquals(List.of("Bob", ""), rows.get(2)); + assertEquals(3, rows.size()); + } + + @Test + void parsesASemicolonFile() throws Exception { + SalesforceBulkCsv.Builder builder = + new SalesforceBulkCsv.Builder( + List.of("Name"), SalesforceBulkDelimiter.SEMICOLON, SalesforceBulkLineEnding.CRLF); + builder.add(List.of("A;B")); + List> rows = + SalesforceBulkCsv.parse( + new String(builder.toByteArray(), StandardCharsets.UTF_8), + SalesforceBulkDelimiter.SEMICOLON); + assertEquals("A;B", rows.get(1).get(0)); + } +} diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClientTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClientTest.java new file mode 100644 index 00000000000..a083fbee837 --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkHttpClientTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.patch; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.put; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import org.apache.hop.core.exception.HopException; +import org.junit.jupiter.api.Test; + +@WireMockTest +class SalesforceBulkHttpClientTest { + + @Test + void runsAnIngestJobAndAPagedQuery(WireMockRuntimeInfo wireMock) throws Exception { + stubFor( + post(urlEqualTo("/services/data/v64.0/jobs/ingest")) + .withHeader("Authorization", equalTo("Bearer test-token")) + .willReturn(json("{\"id\":\"7501\",\"state\":\"Open\"}"))); + stubFor( + put(urlEqualTo("/services/data/v64.0/jobs/ingest/7501/batches")) + .willReturn(aResponse().withStatus(201))); + stubFor( + patch(urlEqualTo("/services/data/v64.0/jobs/ingest/7501")) + .willReturn( + json( + "{\"id\":\"7501\",\"state\":\"JobComplete\",\"numberRecordsProcessed\":1,\"numberRecordsFailed\":0}"))); + stubFor( + get(urlEqualTo("/services/data/v64.0/jobs/ingest/7501/successfulResults")) + .willReturn(aResponse().withStatus(200).withBody("Name,sf__Id\nAda,001\n"))); + stubFor( + get(urlEqualTo("/services/data/v64.0/jobs/ingest/7501/failedResults")) + .willReturn(aResponse().withStatus(404))); + stubFor( + post(urlEqualTo("/services/data/v64.0/jobs/query")) + .willReturn(json("{\"id\":\"750q\",\"state\":\"UploadComplete\"}"))); + stubFor( + get(urlEqualTo("/services/data/v64.0/jobs/query/750q")) + .willReturn(json("{\"id\":\"750q\",\"state\":\"JobComplete\"}"))); + stubFor( + get(urlEqualTo("/services/data/v64.0/jobs/query/750q/results")) + .withHeader("Sforce-Locator", absent()) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Sforce-Locator", "page2") + .withBody("Id,Name\n1,Acme\n"))); + stubFor( + get(urlEqualTo("/services/data/v64.0/jobs/query/750q/results")) + .withHeader("Sforce-Locator", equalTo("page2")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Sforce-Locator", "null") + .withBody("Id,Name\n2,Beta\n"))); + + SalesforceBulkSession session = + new SalesforceBulkSession(wireMock.getHttpBaseUrl(), "test-token", "64.0"); + try (SalesforceBulkHttpClient client = new SalesforceBulkHttpClient(session)) { + SalesforceBulkJob created = + client.createIngestJob( + new SalesforceBulkIngestRequest( + "Account", + SalesforceBulkOperation.INSERT, + null, + SalesforceBulkLineEnding.LF, + SalesforceBulkDelimiter.COMMA, + SalesforceBulkConcurrency.PARALLEL)); + assertEquals("7501", created.getId()); + client.uploadIngestData("7501", "Name\nAda\n".getBytes()); + SalesforceBulkJob done = client.completeIngestUpload("7501"); + assertEquals("JobComplete", done.getState()); + assertEquals(1, done.getRecordsProcessed()); + assertTrue(client.ingestResults("7501", false).contains("Ada")); + assertEquals("", client.ingestResults("7501", true)); + + SalesforceBulkJob query = client.createQueryJob("query", "SELECT Id, Name FROM Account"); + SalesforceBulkJob finished = client.getQueryJob(query.getId()); + assertTrue(finished.isFinished()); + SalesforceBulkQueryPage first = client.queryResults(query.getId(), null); + assertTrue(first.hasMore()); + assertTrue(first.getCsv().contains("Acme")); + SalesforceBulkQueryPage second = client.queryResults(query.getId(), first.getLocator()); + assertFalse(second.hasMore()); + assertTrue(second.getCsv().contains("Beta")); + } + } + + @Test + void reportsAnUnauthorizedCall(WireMockRuntimeInfo wireMock) { + stubFor( + post(urlEqualTo("/services/data/v64.0/jobs/ingest")) + .willReturn( + aResponse().withStatus(401).withBody("[{\"errorCode\":\"INVALID_SESSION_ID\"}]"))); + SalesforceBulkHttpClient client = + new SalesforceBulkHttpClient( + new SalesforceBulkSession(wireMock.getHttpBaseUrl(), "bad", "64.0")); + HopException error = + assertThrows( + HopException.class, + () -> + client.createIngestJob( + new SalesforceBulkIngestRequest( + "Account", + SalesforceBulkOperation.INSERT, + null, + SalesforceBulkLineEnding.LF, + SalesforceBulkDelimiter.COMMA, + SalesforceBulkConcurrency.PARALLEL))); + assertTrue(error.getMessage().contains("401")); + client.close(); + } + + private static ResponseDefinitionBuilder json(String body) { + return aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(body); + } +} diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadTest.java new file mode 100644 index 00000000000..833bae309f6 --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkLoadTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class SalesforceBulkLoadTest { + + @BeforeAll + static void setUpHop() throws Exception { + HopEnvironment.init(); + } + + @Test + void returnsRowsInInputOrderWhenSalesforceReversesTheFile() throws Exception { + SalesforceBulkLoadMeta meta = new SalesforceBulkLoadMeta(); + meta.setSalesforceObject("Account"); + meta.setOperation(SalesforceBulkOperation.INSERT); + meta.setFields(List.of(new SalesforceBulkField("Name", "Name"))); + meta.setMaxBytes("1000000"); + SalesforceBulkLoadData data = new SalesforceBulkLoadData(); + data.client = new EchoClient(); + TransformMeta transformMeta = new TransformMeta("load", meta); + PipelineMeta pipelineMeta = new PipelineMeta(); + pipelineMeta.setName("bulk"); + pipelineMeta.addTransform(transformMeta); + Harness transform = new Harness(transformMeta, meta, data, pipelineMeta); + while (transform.processRow()) { + // Drain the three input rows and the final flush. + } + assertEquals(3, transform.output.size()); + assertEquals("001000000000001AAA", transform.output.get(0)[1]); + assertEquals(true, transform.output.get(0)[3]); + assertEquals("001000000000002AAA", transform.output.get(1)[1]); + assertEquals(false, transform.output.get(2)[3]); + assertEquals("simulated failure", transform.output.get(2)[4]); + transform.dispose(); + } + + private static final class Harness extends SalesforceBulkLoad { + private final Queue rows = new ArrayDeque<>(); + private final IRowMeta input; + private final List output = new ArrayList<>(); + + private Harness( + TransformMeta transformMeta, + SalesforceBulkLoadMeta meta, + SalesforceBulkLoadData data, + PipelineMeta pipelineMeta) { + super(transformMeta, meta, data, 0, pipelineMeta, null); + input = new RowMeta(); + input.addValueMeta(new ValueMetaString("Name")); + rows.add(new Object[] {"Alice"}); + rows.add(new Object[] {"Bob"}); + rows.add(new Object[] {"FAIL"}); + } + + @Override + public Object[] getRow() { + return rows.poll(); + } + + @Override + public IRowMeta getInputRowMeta() { + return input; + } + + @Override + public void putRow(IRowMeta rowMeta, Object[] row) { + output.add(row); + } + } + + private static final class EchoClient implements SalesforceBulkClient { + private byte[] uploaded; + + @Override + public SalesforceBulkJob createIngestJob(SalesforceBulkIngestRequest request) { + return new SalesforceBulkJob("7501", "Open", null, 0, 0); + } + + @Override + public void uploadIngestData(String jobId, byte[] csv) { + uploaded = csv; + } + + @Override + public SalesforceBulkJob completeIngestUpload(String jobId) { + return new SalesforceBulkJob(jobId, "UploadComplete", null, 0, 0); + } + + @Override + public SalesforceBulkJob getIngestJob(String jobId) { + return new SalesforceBulkJob(jobId, "JobComplete", null, 3, 1); + } + + @Override + public String ingestResults(String jobId, boolean failed) throws HopException { + List> table = + SalesforceBulkCsv.parse(new String(uploaded, StandardCharsets.UTF_8)); + StringBuilder csv = new StringBuilder(); + csv.append(failed ? "Name,sf__Id,sf__Error\n" : "Name,sf__Id,sf__Created\n"); + List> data = new ArrayList<>(table.subList(1, table.size())); + java.util.Collections.reverse(data); + for (List row : data) { + boolean fail = "FAIL".equals(row.get(0)); + if (failed != fail) { + continue; + } + if (fail) { + csv.append("FAIL,,simulated failure\n"); + } else if ("Alice".equals(row.get(0))) { + csv.append("Alice,001000000000001AAA,true\n"); + } else { + csv.append("Bob,001000000000002AAA,true\n"); + } + } + return csv.toString(); + } + + @Override + public void abortIngestJob(String jobId) {} + + @Override + public SalesforceBulkJob createQueryJob(String operation, String soql) { + throw new UnsupportedOperationException(); + } + + @Override + public SalesforceBulkJob getQueryJob(String jobId) { + throw new UnsupportedOperationException(); + } + + @Override + public SalesforceBulkQueryPage queryResults(String jobId, String locator) { + throw new UnsupportedOperationException(); + } + + @Override + public void abortQueryJob(String jobId) {} + + @Override + public void close() {} + } +} diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcherTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcherTest.java new file mode 100644 index 00000000000..e584a228aeb --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkRowMatcherTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class SalesforceBulkRowMatcherTest { + + @Test + void matchesResultsThatAreNotInInputOrder() throws Exception { + List pending = new ArrayList<>(); + pending.add(row("Alice")); + pending.add(row("Bob")); + pending.add(row("FAIL")); + String success = + "Name,sf__Id,sf__Created\nBob,001000000000002AAA,true\nAlice,001000000000001AAA,true\n"; + String failed = "Name,sf__Id,sf__Error\nFAIL,,simulated failure\n"; + SalesforceBulkRowMatcher.match( + pending, List.of("Name"), new int[] {0}, success, SalesforceBulkDelimiter.COMMA, false); + SalesforceBulkRowMatcher.match( + pending, List.of("Name"), new int[] {0}, failed, SalesforceBulkDelimiter.COMMA, true); + + assertTrue(pending.get(0).isMatched()); + assertEquals("001000000000001AAA", pending.get(0).getSalesforceId()); + assertEquals("001000000000002AAA", pending.get(1).getSalesforceId()); + assertEquals("simulated failure", pending.get(2).getError()); + assertTrue(pending.get(2).isMatched()); + } + + @Test + void pairsDuplicateKeysInInputOrder() throws Exception { + List pending = new ArrayList<>(); + pending.add(row("Ada")); + pending.add(row("Ada")); + String success = "Name,sf__Id\nAda,second\nAda,first\n"; + SalesforceBulkRowMatcher.match( + pending, List.of("Name"), new int[] {0}, success, SalesforceBulkDelimiter.COMMA, false); + assertEquals("second", pending.get(0).getSalesforceId()); + assertEquals("first", pending.get(1).getSalesforceId()); + assertFalse(pending.stream().anyMatch(row -> !row.isMatched())); + } + + private static SalesforceBulkRowMatcher.Pending row(String name) { + return new SalesforceBulkRowMatcher.Pending(new Object[] {name}, List.of(name)); + } +} diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSessionTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSessionTest.java new file mode 100644 index 00000000000..7b654c6fe6a --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkSessionTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +import com.sforce.soap.partner.LoginResult; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.pipeline.transforms.salesforce.SalesforceConnection; +import org.junit.jupiter.api.Test; + +class SalesforceBulkSessionTest { + + @Test + void stripsTheSoapPathFromTheServerUrl() { + assertEquals( + "https://na1.salesforce.com", + SalesforceBulkSession.instanceRoot("https://na1.salesforce.com/services/Soap/u/64.0")); + assertEquals( + "http://salesforce_mock:8080", + SalesforceBulkSession.instanceRoot("http://salesforce_mock:8080/")); + } + + @Test + void usesTheOauthTokenWithoutALogin() throws Exception { + SalesforceConnection connection = + new SalesforceConnection( + mock(ILogChannel.class), "client", "secret", "token", "http://salesforce_mock:8080/"); + SalesforceBulkSession session = SalesforceBulkSession.from(connection); + assertEquals("token", session.getAccessToken()); + assertEquals("http://salesforce_mock:8080", session.getInstanceUrl()); + assertEquals("64.0", session.getApiVersion()); + } + + @Test + void usesTheSoapSessionIdAsABearerToken() throws Exception { + SalesforceConnection connection = + new SalesforceConnection( + mock(ILogChannel.class), + "https://login.salesforce.com/services/Soap/u/64.0", + "user", + "secret"); + LoginResult login = new LoginResult(); + login.setSessionId("session-id"); + login.setServerUrl("https://na1.salesforce.com/services/Soap/u/64.0"); + connection.setLoginResult(login); + SalesforceBulkSession session = SalesforceBulkSession.from(connection); + assertEquals("session-id", session.getAccessToken()); + assertEquals("https://na1.salesforce.com", session.getInstanceUrl()); + } +} diff --git a/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValuesTest.java b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValuesTest.java new file mode 100644 index 00000000000..2ef5802c5e2 --- /dev/null +++ b/plugins/tech/salesforce/src/test/java/org/apache/hop/pipeline/transforms/salesforcebulk/SalesforceBulkValuesTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.salesforcebulk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Calendar; +import java.util.Date; +import java.util.TimeZone; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.junit.jupiter.api.Test; + +class SalesforceBulkValuesTest { + + @Test + void writesNullAsEmptyOrNa() throws Exception { + ValueMetaString meta = new ValueMetaString("Name"); + assertEquals("", SalesforceBulkValues.format(meta, null, false)); + assertEquals("#N/A", SalesforceBulkValues.format(meta, null, true)); + assertEquals("Ada", SalesforceBulkValues.format(meta, "Ada", true)); + } + + @Test + void writesNumbersWithoutGrouping() throws Exception { + ValueMetaNumber meta = new ValueMetaNumber("Amount"); + assertEquals("1234.5", SalesforceBulkValues.format(meta, 1234.5d, false)); + ValueMetaInteger integer = new ValueMetaInteger("Count"); + assertEquals("42", SalesforceBulkValues.format(integer, 42L, false)); + } + + @Test + void writesDatesInUtc() throws Exception { + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + calendar.clear(); + calendar.set(2024, Calendar.MARCH, 2, 4, 5, 6); + calendar.set(Calendar.MILLISECOND, 7); + Date date = calendar.getTime(); + assertEquals( + "2024-03-02T04:05:06.007Z", + SalesforceBulkValues.format(new ValueMetaDate("When"), date, false)); + assertEquals( + date, SalesforceBulkValues.parse(new ValueMetaDate("When"), "2024-03-02T04:05:06.007Z")); + } + + @Test + void readsAnEmptyQueryFieldAsNull() throws Exception { + assertNull(SalesforceBulkValues.parse(new ValueMetaString("Name"), "")); + } +}