diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 18aaa40cf4fd..67c9e446f392 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -94,6 +94,8 @@ Beam Jenkins overview page: [link](https://builds.apache.org/view/A-D/view/Beam/ |------|------|-------------------|-------------| | beam_PerformanceTests_Analysis | [cron](https://builds.apache.org/job/beam_PerformanceTests_Analysis/) | `Run Performance Tests Analysis` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Analysis/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Analysis) | | beam_PerformanceTests_AvroIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS/) | `Run Java AvroIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS) | +| beam_PerformanceTests_BiqQueryIO_Read_Python | [cron](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python/) | `Run BigQueryIO Read Performance Test Python` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python) | +| beam_PerformanceTests_BiqQueryIO_Write_Python_Batch | [cron](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch/) | `Run BigQueryIO Write Performance Test Python Batch` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch) | | beam_PerformanceTests_Compressed_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS/) | `Run Java CompressedTextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS) | | beam_PerformanceTests_HadoopFormat | [cron](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat/) | `Run Java HadoopFormatIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat) | | beam_PerformanceTests_JDBC | [cron](https://builds.apache.org/job/beam_PerformanceTests_JDBC/) | `Run Java JdbcIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_JDBC/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_JDBC) | @@ -101,6 +103,7 @@ Beam Jenkins overview page: [link](https://builds.apache.org/view/A-D/view/Beam/ | beam_PerformanceTests_ManyFiles_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS/) | `Run Java ManyFilesTextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS) | | beam_PerformanceTests_MongoDBIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT/) | `Run Java MongoDBIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT) | | beam_PerformanceTests_ParquetIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS/) | `Run Java ParquetIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS) | +| beam_PerformanceTests_PubsubIOIT_Python_Streaming | [cron](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming/) | `Run PubsubIO Performance Test Python` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming) | | beam_PerformanceTests_Spark | [cron](https://builds.apache.org/job/beam_PerformanceTests_Spark/) | `Run Spark Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Spark/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Spark) | | beam_PerformanceTests_TFRecordIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT/) | `Run Java TFRecordIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT) | | beam_PerformanceTests_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/) | `Run Java TextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS) | diff --git a/.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy index c8fcc6825344..56dc0c894192 100644 --- a/.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy @@ -82,7 +82,7 @@ def executeJob = { scope, testConfig -> } PhraseTriggeringPostCommitBuilder.postCommitJob( - 'beam_BiqQueryIO_Read_Performance_Test_Python', + 'beam_PerformanceTests_BiqQueryIO_Read_Python', 'Run BigQueryIO Read Performance Test Python', 'BigQueryIO Read Performance Test Python', this @@ -90,12 +90,12 @@ PhraseTriggeringPostCommitBuilder.postCommitJob( executeJob(delegate, bqio_read_test) } -CronJobBuilder.cronJob('beam_BiqQueryIO_Read_Performance_Test_Python', 'H 15 * * *', this) { +CronJobBuilder.cronJob('beam_PerformanceTests_BiqQueryIO_Read_Python', 'H 15 * * *', this) { executeJob(delegate, bqio_read_test) } PhraseTriggeringPostCommitBuilder.postCommitJob( - 'beam_BiqQueryIO_Write_Performance_Test_Python_Batch', + 'beam_PerformanceTests_BiqQueryIO_Write_Python_Batch', 'Run BigQueryIO Write Performance Test Python Batch', 'BigQueryIO Write Performance Test Python Batch', this @@ -103,6 +103,6 @@ PhraseTriggeringPostCommitBuilder.postCommitJob( executeJob(delegate, bqio_write_test) } -CronJobBuilder.cronJob('beam_BiqQueryIO_Write_Performance_Test_Python_Batch', 'H 15 * * *', this) { +CronJobBuilder.cronJob('beam_PerformanceTests_BiqQueryIO_Write_Python_Batch', 'H 15 * * *', this) { executeJob(delegate, bqio_write_test) } diff --git a/.test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy index 5ca1458c4fcd..f08e53ec765d 100644 --- a/.test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy @@ -39,7 +39,10 @@ def psio_test = [ temp_location : 'gs://temp-storage-for-perf-tests/loadtests', publish_to_big_query : true, metrics_dataset : 'beam_performance', - metrics_table : 'psio_io_2GB_msg_results', + metrics_table : 'psio_io_2GB_results', + influx_measurement : 'python_psio_2GB_results', + influx_db_name : InfluxDBCredentialsHelper.InfluxDBDatabaseName, + influx_hostname : InfluxDBCredentialsHelper.InfluxDBHostname, input_options : '\'{' + '"num_records": 2097152,' + '"key_size": 1,' + @@ -59,7 +62,7 @@ def executeJob = { scope, testConfig -> } PhraseTriggeringPostCommitBuilder.postCommitJob( - 'beam_PubsubIO_Performance_Test_Python', + 'beam_PerformanceTests_PubsubIOIT_Python_Streaming', 'Run PubsubIO Performance Test Python', 'PubsubIO Performance Test Python', this @@ -67,6 +70,6 @@ PhraseTriggeringPostCommitBuilder.postCommitJob( executeJob(delegate, psio_test) } -CronJobBuilder.cronJob('beam_PubsubIO_Performance_Test_Python', 'H 15 * * *', this) { +CronJobBuilder.cronJob('beam_PerformanceTests_PubsubIOIT_Python_Streaming', 'H 15 * * *', this) { executeJob(delegate, psio_test) } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index f25f133c282e..bd5888f42df3 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -39,5 +39,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python35') shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python36') shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python37') + // TODO(BEAM-9754): Turn on ValidatesContainer tests on Python 3.8 once BEAM-9754 is resolved. + // shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python38') } } diff --git a/.test-infra/metrics/beamgrafana-deploy.yaml b/.test-infra/metrics/beamgrafana-deploy.yaml index 8a4ecb7a474a..7c5a83dd6784 100644 --- a/.test-infra/metrics/beamgrafana-deploy.yaml +++ b/.test-infra/metrics/beamgrafana-deploy.yaml @@ -37,7 +37,7 @@ spec: fsGroup: 1000 containers: - name: beamgrafana - image: gcr.io/apache-beam-testing/beamgrafana:beammetrics20200515 + image: gcr.io/apache-beam-testing/beamgrafana:beammetrics20200526 securityContext: runAsUser: 0 env: diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_IO_IT_Tests_Dataflow.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_IO_IT_Tests_Dataflow.json index 60d9a86dba56..c7bf2e390862 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_IO_IT_Tests_Dataflow.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_IO_IT_Tests_Dataflow.json @@ -35,7 +35,7 @@ }, "hiddenSeries": false, "id": 2, - "interval": "", + "interval": "24h", "legend": { "avg": false, "current": false, @@ -75,7 +75,7 @@ "measurement": "python_bqio_read", "orderByTime": "ASC", "policy": "default", - "query": "SELECT value FROM \"python_bqio_read_10GB_results\" WHERE \"metric\" =~ /runtime/", + "query": "SELECT mean(\"value\") FROM \"python_bqio_read_10GB_results\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -157,7 +157,7 @@ }, "hiddenSeries": false, "id": 3, - "interval": "", + "interval": "24h", "legend": { "avg": false, "current": false, @@ -197,7 +197,7 @@ "measurement": "python_bqio_read", "orderByTime": "ASC", "policy": "default", - "query": "SELECT value FROM \"python_bqio_write_10GB_results\" WHERE \"metric\" =~ /runtime/", + "query": "SELECT mean(\"value\") FROM \"python_bqio_write_10GB_results\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -261,6 +261,250 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "BeamInfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 4, + "interval": "24h", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.7.2", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "read_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "measurement": "python_bqio_read", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"python_psio_2GB_results\" WHERE \"metric\" = 'pubsub_io_perf_read_runtime' AND $timeFilter GROUP BY time($__interval), \"metric\"", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Reading 2GB of data | Pubsub native Dataflow IO | streaming", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:403", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:404", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "BeamInfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 5, + "interval": "24h", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.7.2", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "write_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "measurement": "python_bqio_read", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"python_psio_2GB_results\" WHERE \"metric\" = 'pubsub_io_perf_write_runtime' AND $timeFilter GROUP BY time($__interval), \"metric\"", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Writing 2GB of data | Pubsub native Dataflow IO | streaming", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:403", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:404", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "schemaVersion": 22, @@ -293,5 +537,5 @@ "variables": { "list": [] }, - "version": 1 -} + "version": 2 +} \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 9aff55608c1d..d00b9c7fa1e9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,7 +47,7 @@ * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). --> -# [2.22.0] - Unreleased +# [2.23.0] - Unreleased ## Highlights @@ -57,12 +57,48 @@ ## I/Os * Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* Support for reading from Snowflake added (Java) ([BEAM-9722](https://issues.apache.org/jira/browse/BEAM-9722)). + +## New Features / Improvements + +* X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Breaking Changes + +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Deprecations + +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Known Issues + +* Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +# [2.22.0] - Unreleased + +## Highlights + +* New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* New highly anticipated feature Y added to Java SDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + +## I/Os + +* Basic Kafka read/write support for DataflowRunner (Python) ([BEAM-8019](https://issues.apache.org/jira/browse/BEAM-8019)). +* Sources and sinks for Google Healthcare APIs (Java)([BEAM-9468](https://issues.apache.org/jira/browse/BEAM-9468)). ## New Features / Improvements * `--workerCacheMB` flag is supported in Dataflow streaming pipeline ([BEAM-9964](https://issues.apache.org/jira/browse/BEAM-9964)) * `--direct_num_workers=0` is supported for FnApi runner. It will set the number of threads/subprocesses to number of cores of the machine executing the pipeline ([BEAM-9443](https://issues.apache.org/jira/browse/BEAM-9443)). * Python SDK now has experimental support for SqlTransform ([BEAM-8603](https://issues.apache.org/jira/browse/BEAM-8603)). +* Add OnWindowExpiration method to Stateful DoFn ([BEAM-1589](https://issues.apache.org/jira/browse/BEAM-1589)). +* Added PTransforms for Google Cloud DLP (Data Loss Prevention) services integration ([BEAM-9723](https://issues.apache.org/jira/browse/BEAM-9723)): + * Inspection of data, + * Deidentification of data, + * Reidentification of data. +* Add a more complete I/O support matrix in the documentation site ([BEAM-9916](https://issues.apache.org/jira/browse/BEAM-9916)). +* Upgrade Sphinx to 3.0.3 for building PyDoc. ## Breaking Changes @@ -76,7 +112,7 @@ * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). -# [2.21.0] - Unreleased (In Progress) +# [2.21.0] - 2020-05-27 ## Highlights @@ -104,7 +140,7 @@ for example usage. More details will be in [Ensuring Python Type Safety](https://beam.apache.org/documentation/sdks/python-type-safety/) and an upcoming - [blog post](https://beam.apache.org/blog/python/typing/2020/03/06/python-typing.html). + [blog post](https://beam.apache.org/blog/python-typing/index.html). * Java SDK: Introducing the concept of options in Beam Schema’s. These options add extra context to fields and schemas. This replaces the current Beam metadata that is present @@ -144,6 +180,7 @@ conversion to beam schema options. *Remark: Schema aware is still experimental.* ## Deprecations * Java SDK: Beam Schema FieldType.getMetadata is now deprecated and is replaced by the Beam Schema Options, it will be removed in version `2.23.0`. ([BEAM-9704](https://issues.apache.org/jira/browse/BEAM-9704)) +* The `--zone` option in the Dataflow runner is now deprecated. Please use `--worker_zone` instead. ([BEAM-9716](https://issues.apache.org/jira/browse/BEAM-9716)) ## Known Issues diff --git a/build.gradle b/build.gradle index ca4b54ea2710..74138fc7c7ad 100644 --- a/build.gradle +++ b/build.gradle @@ -112,6 +112,9 @@ rat { "learning/katas/**/task-remote-info.yaml", "learning/katas/*/IO/**/*.txt", + // test p8 file for SnowflakeIO + "sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8", + // Mockito extensions "sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker", "sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker" @@ -152,6 +155,8 @@ task javaPreCommit() { } task sqlPreCommit() { + dependsOn ":sdks:java:extensions:sql:runBasicExample" + dependsOn ":sdks:java:extensions:sql:runPojoExample" dependsOn ":sdks:java:extensions:sql:build" dependsOn ":sdks:java:extensions:sql:buildDependents" } @@ -210,6 +215,7 @@ task pythonPreCommit() { dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35" dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36" dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37" + dependsOn ":sdks:python:test-suites:tox:py38:preCommitPy38" dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT" dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT_V2" dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT" @@ -224,6 +230,7 @@ task pythonDockerBuildPreCommit() { dependsOn ":sdks:python:container:py35:docker" dependsOn ":sdks:python:container:py36:docker" dependsOn ":sdks:python:container:py37:docker" + dependsOn ":sdks:python:container:py38:docker" } task pythonLintPreCommit() { @@ -232,7 +239,7 @@ task pythonLintPreCommit() { } task pythonFormatterPreCommit() { - dependsOn 'sdks:python:test-suites:tox:py37:formatter' + dependsOn 'sdks:python:test-suites:tox:py38:formatter' } task python2PostCommit() { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index eaf79fbdd696..8ba8a4d12bb4 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -257,6 +257,12 @@ class BeamModulePlugin implements Plugin { // includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' // excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' } + // Tests to include/exclude from running, by default all tests are included + Closure testFilter = { + // Use the following to include / exclude tests: + // includeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' + // excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' + } // Configuration for the classpath when running the test. Configuration testClasspathConfiguration // Additional system properties. @@ -311,7 +317,7 @@ class BeamModulePlugin implements Plugin { // Automatically use the official release version if we are performing a release // otherwise append '-SNAPSHOT' - project.version = '2.22.0' + project.version = '2.23.0' if (!isRelease(project)) { project.version += '-SNAPSHOT' } @@ -382,7 +388,7 @@ class BeamModulePlugin implements Plugin { def generated_grpc_ga_version = "1.85.1" def google_auth_version = "0.19.0" def google_clients_version = "1.30.9" - def google_cloud_bigdataoss_version = "2.1.2" + def google_cloud_bigdataoss_version = "2.1.3" def google_cloud_core_version = "1.92.2" def google_cloud_spanner_version = "1.49.1" def google_cloud_datacatalog_version = "0.32.1" @@ -481,6 +487,7 @@ class BeamModulePlugin implements Plugin { google_oauth_client : "com.google.oauth-client:google-oauth-client:$google_oauth_clients_version", google_oauth_client_java6 : "com.google.oauth-client:google-oauth-client-java6:$google_oauth_clients_version", grpc_all : "io.grpc:grpc-all:$grpc_version", + grpc_alts : "io.grpc:grpc-alts:$grpc_version", grpc_auth : "io.grpc:grpc-auth:$grpc_version", grpc_core : "io.grpc:grpc-core:$grpc_version", grpc_context : "io.grpc:grpc-context:$grpc_version", @@ -489,6 +496,7 @@ class BeamModulePlugin implements Plugin { grpc_protobuf : "io.grpc:grpc-protobuf:$grpc_version", grpc_protobuf_lite : "io.grpc:grpc-protobuf-lite:$grpc_version", grpc_netty : "io.grpc:grpc-netty:$grpc_version", + grpc_netty_shaded : "io.grpc:grpc-netty-shaded:$grpc_version", grpc_stub : "io.grpc:grpc-stub:$grpc_version", guava : "com.google.guava:guava:$guava_version", guava_testlib : "com.google.guava:guava-testlib:$guava_version", @@ -1687,6 +1695,7 @@ class BeamModulePlugin implements Plugin { testClassesDirs = project.files(project.project(":sdks:java:core").sourceSets.test.output.classesDirs, project.project(":runners:core-java").sourceSets.test.output.classesDirs) maxParallelForks config.numParallelTests useJUnit(config.testCategories) + filter(config.testFilter) // increase maxHeapSize as this is directly correlated to direct memory, // see https://issues.apache.org/jira/browse/BEAM-6698 maxHeapSize = '4g' diff --git a/gradle.properties b/gradle.properties index e29a95f52554..22f2c251bd7e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,8 +23,8 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.22.0-SNAPSHOT -sdk_version=2.22.0.dev +version=2.23.0-SNAPSHOT +sdk_version=2.23.0.dev javaVersion=1.8 diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/cmd/main.go b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/cmd/main.go new file mode 100644 index 000000000000..ee11b3b65a5e --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/cmd/main.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "cogroupbykey/pkg/task" + "context" + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/log" + "github.com/apache/beam/sdks/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/go/pkg/beam/x/debug" +) + +func main() { + ctx := context.Background() + + p, s := beam.NewPipelineWithRoot() + + fruits := beam.Create(s.Scope("Fruits"), "apple", "banana", "cherry") + countries := beam.Create(s.Scope("Countries"), "australia", "brazil", "canada") + + output := task.ApplyTransform(s, fruits, countries) + + debug.Print(s, output) + + err := beamx.Run(ctx, p) + + if err != nil { + log.Exitf(ctx, "Failed to execute job: %v", err) + } +} diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.mod b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.mod new file mode 100644 index 000000000000..c0c99f3de73b --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.mod @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module cogroupbykey + +go 1.14 + +require ( + github.com/apache/beam v2.20.0+incompatible + github.com/golang/protobuf v1.4.2 // indirect + github.com/google/go-cmp v0.4.1 // indirect + google.golang.org/api v0.25.0 // indirect + google.golang.org/grpc v1.29.1 // indirect +) diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.sum b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.sum new file mode 100644 index 000000000000..92a428884f45 --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/go.sum @@ -0,0 +1,314 @@ +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/apache/beam v2.20.0+incompatible h1:YzP/+VyAnYdu4Wjh5EkBz3vUjVdE7vUPEZ6xijCJ2sk= +github.com/apache/beam v2.20.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 h1:MRHtG0U6SnaUb+s+LhNE1qt1FQ1wlhqr5E4usBKC0uA= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/pkg/task/task.go b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/pkg/task/task.go new file mode 100644 index 000000000000..c498b3585a1d --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/pkg/task/task.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "fmt" + "github.com/apache/beam/sdks/go/pkg/beam" +) + +func ApplyTransform(s beam.Scope, fruits beam.PCollection, countries beam.PCollection) beam.PCollection { + fruitsKV := beam.ParDo(s, func(word string) (string, string) { + return string(word[0]), word + }, fruits) + + countriesKV := beam.ParDo(s, func(word string) (string, string) { + return string(word[0]), word + }, countries) + + grouped := beam.CoGroupByKey(s, fruitsKV, countriesKV) + return beam.ParDo(s, func(key string, fruitsIter func(*string) bool, countriesIter func(*string) bool, emit func(string)) { + wa := &WordsAlphabet{ + Alphabet: key, + } + fruitsIter(&wa.Fruit) + countriesIter(&wa.Country) + emit(wa.String()) + }, grouped) +} + +type WordsAlphabet struct { + Alphabet string + Fruit string + Country string +} + +func (wa *WordsAlphabet) String() string { + return fmt.Sprintf("WordsAlphabet%+v", *wa) +} + diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-info.yaml b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-info.yaml new file mode 100644 index 000000000000..2aa604763256 --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-info.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +type: edu +files: +- name: test/task_test.go + visible: false +- name: cmd/main.go + visible: true +- name: go.mod + visible: false +- name: pkg/task/task.go + visible: true + placeholders: + - offset: 978 + length: 533 + placeholder_text: TODO() +- name: go.sum + visible: false diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-remote-info.yaml b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-remote-info.yaml new file mode 100644 index 000000000000..6846e3ac7a55 --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task-remote-info.yaml @@ -0,0 +1,2 @@ +id: 1342650 +update_date: Thu, 28 May 2020 22:33:24 UTC diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task.md b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task.md new file mode 100644 index 000000000000..9d2868abc44c --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/task.md @@ -0,0 +1,104 @@ + + +# CoGroupByKey + +CoGroupByKey performs a relational join of two or more key/value PCollections that have the same +key type. + +**Kata:** Implement a [beam.CoGroupByKey](https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam#CoGroupByKey) +transform that join words by the first alphabetical letter, and then produces the string representation of the +WordsAlphabet model. + +
+ Refer to + beam.CoGroupByKey + to solve this problem. +
+ +
+ Refer to the Beam Programming Guide + + "CoGroupByKey" section for more information. +
+ +
+ Think of this problem in three stages. First, create key/value pairs of PCollections called KV + for fruits and countries, pairing the first character with the word. Next, apply CoGroupByKey to the KVs + followed by a ParDo. +
+ +
+ In the last lesson we learned how to make key/value PCollections called KV. Now we have + two to make from fruits and countries. + + To return as a KV, you can return two values from your DoFn. The first return value represents the Key, and + the second return value represents the Value. An example is shown below. + +``` +func doFn(element string) (string, string) { + key := string(element[0]) + value := element + return key, value +} +``` +
+ +
+ In the last lesson we learned that + + beam.GroupByKey takes a single KV. + beam.CoGroupByKey + takes more than one KV. +
+ +
+ Our final step in this problem requires a + beam.ParDo + with a DoFn that's different than what we've seen in previous lessons. In the previous step we should + have a PCollection acquired from CoGroupByKey. A ParDo for that PCollection expects a DoFn that looks + like the following. + + ``` + func doFn(key string, iterA func(*string) bool, iterB func(*string) bool, emit func(string)){ + ... + } + ``` + + Each `func(*string) bool` parameter above corresponds to each of the PCollection KVs that we provided to CoGroupByKey. + + `func(*string)` returns true if there is a value available when the pipeline invokes your doFn. + + Note that it takes a `*string` instead of a `string`. In Go, this means that to acquire its value, we need to + do this. + +``` + var v string + iterA(&v) + // do something with v +``` + + Not necessary for this task, though if you expected multiple values per key, you would do something like this. +``` + var v string + for iterA(&v) { + // do something with v + } +``` +
diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/test/task_test.go b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/test/task_test.go new file mode 100644 index 000000000000..90a5912ebe22 --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/CoGroupByKey/test/task_test.go @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "cogroupbykey/pkg/task" + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/testing/passert" + "github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" + "testing" +) + +func TestApplyTransform(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + type args struct { + fruits beam.PCollection + countries beam.PCollection + } + tests := []struct { + args args + want []interface{} + }{ + { + args: args{ + fruits: beam.Create(s.Scope("Fruits"), "apple", "banana", "cherry"), + countries: beam.Create(s.Scope("Countries"), "australia", "brazil", "canada"), + }, + want: []interface{}{ + "WordsAlphabet{Alphabet:a Fruit:apple Country:australia}", + "WordsAlphabet{Alphabet:b Fruit:banana Country:brazil}", + "WordsAlphabet{Alphabet:c Fruit:cherry Country:canada}", + }, + }, + } + for _, tt := range tests { + got := task.ApplyTransform(s, tt.args.fruits, tt.args.countries) + passert.Equals(s, got, tt.want...) + if err := ptest.Run(p); err != nil { + t.Error(err) + } + } +} diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/lesson-info.yaml b/learning/katas/go/Core Transforms/CoGroupByKey/lesson-info.yaml new file mode 100644 index 000000000000..273c0771cccf --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/lesson-info.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +content: +- CoGroupByKey diff --git a/learning/katas/go/Core Transforms/CoGroupByKey/lesson-remote-info.yaml b/learning/katas/go/Core Transforms/CoGroupByKey/lesson-remote-info.yaml new file mode 100644 index 000000000000..548d0f7247dc --- /dev/null +++ b/learning/katas/go/Core Transforms/CoGroupByKey/lesson-remote-info.yaml @@ -0,0 +1,3 @@ +id: 361640 +update_date: Thu, 28 May 2020 22:33:20 UTC +unit: 346169 diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/cmd/main.go b/learning/katas/go/Core Transforms/Combine/Simple Function/cmd/main.go new file mode 100644 index 000000000000..eb413972a7b2 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/cmd/main.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/log" + "github.com/apache/beam/sdks/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/go/pkg/beam/x/debug" + "simple_function/pkg/task" +) + +func main() { + ctx := context.Background() + + p, s := beam.NewPipelineWithRoot() + + input := beam.Create(s, 10, 30, 50, 70, 90) + + output := task.ApplyTransform(s, input) + + debug.Print(s, output) + + err := beamx.Run(ctx, p) + + if err != nil { + log.Exitf(context.Background(), "Failed to execute job: %v", err) + } +} diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/go.mod b/learning/katas/go/Core Transforms/Combine/Simple Function/go.mod new file mode 100644 index 000000000000..f8cdb28c1a10 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/go.mod @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module simple_function + +go 1.14 + +require ( + github.com/apache/beam v2.21.0+incompatible + github.com/golang/protobuf v1.4.2 // indirect + github.com/google/go-cmp v0.4.1 // indirect + github.com/googleapis/gax-go v1.0.3 // indirect + google.golang.org/api v0.25.0 // indirect + google.golang.org/grpc v1.29.1 // indirect +) diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/go.sum b/learning/katas/go/Core Transforms/Combine/Simple Function/go.sum new file mode 100644 index 000000000000..00ef75933562 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/go.sum @@ -0,0 +1,310 @@ +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/apache/beam v2.21.0+incompatible h1:ETfDaxjAOSQqy44CyaxqrSW5omoLzmEZmThJEtSLHBI= +github.com/apache/beam v2.21.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go v1.0.3 h1:9dMLqhaibYONnDRcnHdUs9P8Mw64jLlZTYlDe3leBtQ= +github.com/googleapis/gax-go v1.0.3/go.mod h1:QyXYajJFdARxGzjwUfbDFIse7Spkw81SJ4LrBJXtlQ8= +github.com/googleapis/gax-go/v2 v2.0.2/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190221220918-438050ddec5e/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 h1:MRHtG0U6SnaUb+s+LhNE1qt1FQ1wlhqr5E4usBKC0uA= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/pkg/task/task.go b/learning/katas/go/Core Transforms/Combine/Simple Function/pkg/task/task.go new file mode 100644 index 000000000000..afe5abbfa164 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/pkg/task/task.go @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "github.com/apache/beam/sdks/go/pkg/beam" +) + +func ApplyTransform(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.Combine(s, func(sum, elem int) int { + return sum + elem + }, input) +} diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/task-info.yaml b/learning/katas/go/Core Transforms/Combine/Simple Function/task-info.yaml new file mode 100644 index 000000000000..1b3b8e5fd670 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/task-info.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +type: edu +files: +- name: test/task_test.go + visible: false +- name: cmd/main.go + visible: true +- name: go.mod + visible: false +- name: pkg/task/task.go + visible: true + placeholders: + - offset: 949 + length: 72 + placeholder_text: TODO() +- name: go.sum + visible: false diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/task-remote-info.yaml b/learning/katas/go/Core Transforms/Combine/Simple Function/task-remote-info.yaml new file mode 100644 index 000000000000..098280d38d3c --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/task-remote-info.yaml @@ -0,0 +1,2 @@ +id: 1347871 +update_date: Mon, 01 Jun 2020 14:35:52 UTC diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/task.md b/learning/katas/go/Core Transforms/Combine/Simple Function/task.md new file mode 100644 index 000000000000..ea0cc8c52760 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/task.md @@ -0,0 +1,43 @@ + + +# Combine - Simple Function + +Combine is a Beam transform for combining collections of elements or values in your data. When you +apply a Combine transform, you must provide the function that contains the logic for combining the +elements or values. The combining function should be commutative and associative, as the function +is not necessarily invoked exactly once on all values with a given key. Because the input data +(including the value collection) may be distributed across multiple workers, the combining function +might be called multiple times to perform partial combining on subsets of the value collection. + +Simple combine operations, such as sums, can usually be implemented as a simple function. + +**Kata:** Implement the summation of numbers using +[beam.Combine](https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam#Combine). + +
+ Implement a combine function compatible with the + + beam.Combine method that performs the summation of a PCollection<int>. +
+ +
+ Refer to the Beam Programming Guide + + "Simple combinations using simple functions" section for more information. +
diff --git a/learning/katas/go/Core Transforms/Combine/Simple Function/test/task_test.go b/learning/katas/go/Core Transforms/Combine/Simple Function/test/task_test.go new file mode 100644 index 000000000000..2b0ff0c3bdda --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/Simple Function/test/task_test.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/testing/passert" + "github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" + "simple_function/pkg/task" + "testing" +) + +func TestApplyTransform(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + tests := []struct { + input beam.PCollection + want []interface{} + }{ + { + input: beam.Create(s, 10, 30, 50, 70, 90), + want: []interface{}{250}, + }, + } + for _, tt := range tests { + got := task.ApplyTransform(s, tt.input) + passert.Equals(s, got, tt.want...) + if err := ptest.Run(p); err != nil { + t.Error(err) + } + } +} diff --git a/learning/katas/go/Core Transforms/Combine/lesson-info.yaml b/learning/katas/go/Core Transforms/Combine/lesson-info.yaml new file mode 100644 index 000000000000..698fc8cc2265 --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/lesson-info.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +content: +- Simple Function diff --git a/learning/katas/go/Core Transforms/Combine/lesson-remote-info.yaml b/learning/katas/go/Core Transforms/Combine/lesson-remote-info.yaml new file mode 100644 index 000000000000..3747bd27cf4d --- /dev/null +++ b/learning/katas/go/Core Transforms/Combine/lesson-remote-info.yaml @@ -0,0 +1,3 @@ +id: 362667 +update_date: Mon, 01 Jun 2020 14:35:48 UTC +unit: 347283 diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/cmd/main.go b/learning/katas/go/Core Transforms/Flatten/Flatten/cmd/main.go new file mode 100644 index 000000000000..486287241144 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/cmd/main.go @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/log" + "github.com/apache/beam/sdks/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/go/pkg/beam/x/debug" + "context" + "flatten/pkg/task" +) + +func main() { + ctx := context.Background() + + p, s := beam.NewPipelineWithRoot() + + aWords := beam.Create(s, "apple", "ant", "arrow") + bWords := beam.Create(s, "ball", "book", "bow") + + + output := task.ApplyTransform(s, aWords, bWords) + + debug.Print(s, output) + + err := beamx.Run(ctx, p) + + if err != nil { + log.Exitf(context.Background(), "Failed to execute job: %v", err) + } +} diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/go.mod b/learning/katas/go/Core Transforms/Flatten/Flatten/go.mod new file mode 100644 index 000000000000..d0ed83857607 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/go.mod @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module flatten + +go 1.13 + +require ( + github.com/apache/beam v2.20.0+incompatible + github.com/golang/protobuf v1.4.0 // indirect + github.com/google/go-cmp v0.4.0 + github.com/googleapis/gax-go v2.0.2+incompatible // indirect + go.opencensus.io v0.22.3 // indirect + google.golang.org/api v0.22.0 // indirect + google.golang.org/grpc v1.28.1 // indirect +) diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/go.sum b/learning/katas/go/Core Transforms/Flatten/Flatten/go.sum new file mode 100644 index 000000000000..15ecb7727e03 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/go.sum @@ -0,0 +1,123 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/apache/beam v2.20.0+incompatible h1:YzP/+VyAnYdu4Wjh5EkBz3vUjVdE7vUPEZ6xijCJ2sk= +github.com/apache/beam v2.20.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= +google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/pkg/task/task.go b/learning/katas/go/Core Transforms/Flatten/Flatten/pkg/task/task.go new file mode 100644 index 000000000000..bad672e42b20 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/pkg/task/task.go @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import "github.com/apache/beam/sdks/go/pkg/beam" + +func ApplyTransform(s beam.Scope, aInputs beam.PCollection, bInputs beam.PCollection) beam.PCollection { + return beam.Flatten(s, aInputs, bInputs) +} \ No newline at end of file diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/task-info.yaml b/learning/katas/go/Core Transforms/Flatten/Flatten/task-info.yaml new file mode 100644 index 000000000000..4f9e9b55403f --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/task-info.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: edu +files: +- name: go.sum + visible: false +- name: test/task_test.go + visible: false +- name: cmd/main.go + visible: true +- name: go.mod + visible: false +- name: pkg/task/task.go + visible: true + placeholders: + - offset: 972 + length: 33 + placeholder_text: TODO() diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/task-remote-info.yaml b/learning/katas/go/Core Transforms/Flatten/Flatten/task-remote-info.yaml new file mode 100644 index 000000000000..6d24d40ec8c0 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/task-remote-info.yaml @@ -0,0 +1,2 @@ +id: 1342651 +update_date: Thu, 28 May 2020 22:33:30 UTC diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/task.md b/learning/katas/go/Core Transforms/Flatten/Flatten/task.md new file mode 100644 index 000000000000..d1d8a42fb929 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/task.md @@ -0,0 +1,38 @@ + + +Flatten +------- + +Flatten is a Beam transform for PCollection objects that store the same data type. Flatten merges +multiple PCollection objects into a single logical PCollection. + +**Kata:** Implement a +[Flatten](https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam#Flatten) +transform that merges two PCollection of words into a single PCollection. + +
+ Refer to + Flatten to solve this problem. +
+ +
+ Refer to the Beam Programming Guide + + "Flatten" section for more information. +
\ No newline at end of file diff --git a/learning/katas/go/Core Transforms/Flatten/Flatten/test/task_test.go b/learning/katas/go/Core Transforms/Flatten/Flatten/test/task_test.go new file mode 100644 index 000000000000..2615e31f64df --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/Flatten/test/task_test.go @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "flatten/pkg/task" + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/testing/passert" + "github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" + "testing" +) + +func TestApplyTransform(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + tests := []struct { + aWords beam.PCollection + bWords beam.PCollection + want []interface{} + }{ + { + aWords: beam.Create(s, "apple", "ant", "arrow"), + bWords: beam.Create(s, "ball", "book", "bow"), + want: []interface{}{"apple", "ant", "arrow", "ball", "book", "bow"}, + }, + } + for _, tt := range tests { + got := task.ApplyTransform(s, tt.aWords, tt.bWords) + passert.Equals(s, got, tt.want...) + if err := ptest.Run(p); err != nil { + t.Error(err) + } + } +} + diff --git a/learning/katas/go/Core Transforms/Flatten/lesson-info.yaml b/learning/katas/go/Core Transforms/Flatten/lesson-info.yaml new file mode 100644 index 000000000000..e68515874b96 --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/lesson-info.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +content: +- Flatten \ No newline at end of file diff --git a/learning/katas/go/Core Transforms/Flatten/lesson-remote-info.yaml b/learning/katas/go/Core Transforms/Flatten/lesson-remote-info.yaml new file mode 100644 index 000000000000..a6b715b74dda --- /dev/null +++ b/learning/katas/go/Core Transforms/Flatten/lesson-remote-info.yaml @@ -0,0 +1,3 @@ +id: 361641 +update_date: Mon, 01 Jun 2020 14:35:55 UTC +unit: 346170 diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/cmd/main.go b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/cmd/main.go new file mode 100644 index 000000000000..e2c077867af6 --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/cmd/main.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/log" + "github.com/apache/beam/sdks/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/go/pkg/beam/x/debug" + "groupbykey/pkg/task" +) + +func main() { + ctx := context.Background() + + p, s := beam.NewPipelineWithRoot() + + input := beam.Create(s, "apple", "ball", "car", "bear", "cheetah", "ant") + + output := task.ApplyTransform(s, input) + + debug.Print(s, output) + + err := beamx.Run(ctx, p) + + if err != nil { + log.Exitf(context.Background(), "Failed to execute job: %v", err) + } +} diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.mod b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.mod new file mode 100644 index 000000000000..2cb9df14a34d --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.mod @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module groupbykey + +go 1.14 + +require ( + github.com/apache/beam v2.20.0+incompatible + github.com/golang/protobuf v1.4.2 // indirect + github.com/google/go-cmp v0.4.1 + google.golang.org/api v0.24.0 // indirect + google.golang.org/grpc v1.29.1 // indirect +) diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.sum b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.sum new file mode 100644 index 000000000000..eb28a28159c5 --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/go.sum @@ -0,0 +1,315 @@ +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/apache/beam v2.20.0+incompatible h1:YzP/+VyAnYdu4Wjh5EkBz3vUjVdE7vUPEZ6xijCJ2sk= +github.com/apache/beam v2.20.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0 h1:cG03eaksBzhfSIk7JRGctfp3lanklcOM/mTGvow7BbQ= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 h1:MRHtG0U6SnaUb+s+LhNE1qt1FQ1wlhqr5E4usBKC0uA= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/pkg/task/task.go b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/pkg/task/task.go new file mode 100644 index 000000000000..1d9ce6d45c6b --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/pkg/task/task.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import "github.com/apache/beam/sdks/go/pkg/beam" + +func ApplyTransform(s beam.Scope, input beam.PCollection) beam.PCollection { + kv := beam.ParDo(s, func(element string) (string, string) { + return string(element[0]), element + }, input) + return beam.GroupByKey(s, kv) +} diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-info.yaml b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-info.yaml new file mode 100644 index 000000000000..b4aebb3cf48c --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-info.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +type: edu +files: +- name: test/task_test.go + visible: false +- name: cmd/main.go + visible: true +- name: go.mod + visible: false +- name: pkg/task/task.go + visible: true + placeholders: + - offset: 937 + length: 138 + placeholder_text: TODO() +- name: go.sum + visible: false diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-remote-info.yaml b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-remote-info.yaml new file mode 100644 index 000000000000..a766ed47f2b3 --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task-remote-info.yaml @@ -0,0 +1,2 @@ +id: 1325218 +update_date: Wed, 20 May 2020 06:15:19 UTC diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task.md b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task.md new file mode 100644 index 000000000000..66ce7c896d0a --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/task.md @@ -0,0 +1,54 @@ + + +# GroupByKey + +GroupByKey is a Beam transform for processing collections of key/value pairs. It’s a parallel +reduction operation, analogous to the Shuffle phase of a Map/Shuffle/Reduce-style algorithm. The +input to GroupByKey is a collection of key/value pairs that represents a multimap, where the +collection contains multiple pairs that have the same key, but different values. Given such a +collection, you use GroupByKey to collect all of the values associated with each unique key. + +**Kata:** Implement a + +beam.GroupByKey transform that groups words by its first letter. + +
+ Refer to + beam.GroupByKey to solve this problem. +
+ +
+ Refer to the Beam Programming Guide + + "GroupByKey" section for more information. +
+ +
+ To return as a KV, you can return two values from your DoFn. The first return value represents the Key, and + the second return value represents the Value. An example is shown below. + +``` +func doFn(element string) (string, string) { + key := string(element[0]) + value := element + return key, value +} +``` +
diff --git a/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/test/task_test.go b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/test/task_test.go new file mode 100644 index 000000000000..7f9a8827a984 --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/GroupByKey/test/task_test.go @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" + "github.com/google/go-cmp/cmp" + "groupbykey/pkg/task" + "testing" +) + +func TestApplyTransform(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + tests := []struct { + input beam.PCollection + want map[string][]string + }{ + { + input: beam.Create(s, "apple", "ball", "car", "bear", "cheetah", "ant"), + want: map[string][]string{ + "a": {"apple", "ant"}, + "b": {"ball", "bear"}, + "c": {"car", "cheetah"}, + }, + }, + } + for _, tt := range tests { + got := task.ApplyTransform(s, tt.input) + beam.ParDo0(s, func(key string, values func(*string) bool) { + var got []string + var v string + for values(&v) { + got = append(got, v) + } + if !cmp.Equal(got, tt.want[key]) { + t.Errorf("ApplyTransform() key = %s, got %v , want %v", key, got, tt.want[key]) + } + }, got) + if err := ptest.Run(p); err != nil { + t.Error(err) + } + } +} diff --git a/learning/katas/go/Core Transforms/GroupByKey/lesson-info.yaml b/learning/katas/go/Core Transforms/GroupByKey/lesson-info.yaml new file mode 100644 index 000000000000..5de9eb69f819 --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/lesson-info.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +content: +- GroupByKey diff --git a/learning/katas/go/Core Transforms/GroupByKey/lesson-remote-info.yaml b/learning/katas/go/Core Transforms/GroupByKey/lesson-remote-info.yaml new file mode 100644 index 000000000000..6391a8192a9c --- /dev/null +++ b/learning/katas/go/Core Transforms/GroupByKey/lesson-remote-info.yaml @@ -0,0 +1,3 @@ +id: 358048 +update_date: Wed, 20 May 2020 06:15:15 UTC +unit: 342241 diff --git a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.mod b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.mod index 64758068e63d..89da9b4bcbf4 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.mod +++ b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.mod @@ -15,12 +15,11 @@ module pardo_onetomany -go 1.13 +go 1.14 require ( github.com/apache/beam v2.20.0+incompatible github.com/golang/protobuf v1.4.0 // indirect - github.com/googleapis/gax-go v2.0.2+incompatible // indirect go.opencensus.io v0.22.3 // indirect google.golang.org/api v0.22.0 // indirect google.golang.org/grpc v1.28.1 // indirect diff --git a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.sum b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.sum index 66ee82f3d108..709c1101db07 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.sum +++ b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/go.sum @@ -35,10 +35,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -82,6 +81,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -102,6 +102,7 @@ google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/task-remote-info.yaml b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/task-remote-info.yaml index 2512ec8b7de9..6b6405a4ad16 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo OneToMany/task-remote-info.yaml +++ b/learning/katas/go/Core Transforms/Map/ParDo OneToMany/task-remote-info.yaml @@ -1,2 +1,2 @@ id: 1289189 -update_date: Tue, 05 May 2020 02:01:57 UTC +update_date: Thu, 28 May 2020 22:33:42 UTC diff --git a/learning/katas/go/Core Transforms/Map/ParDo Struct/go.mod b/learning/katas/go/Core Transforms/Map/ParDo Struct/go.mod index 0c0fddeae56e..7b5d3a2e31f7 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo Struct/go.mod +++ b/learning/katas/go/Core Transforms/Map/ParDo Struct/go.mod @@ -15,12 +15,11 @@ module mapelements -go 1.13 +go 1.14 require ( github.com/apache/beam v2.20.0+incompatible github.com/golang/protobuf v1.4.0 // indirect - github.com/googleapis/gax-go v2.0.2+incompatible // indirect go.opencensus.io v0.22.3 // indirect google.golang.org/api v0.22.0 // indirect google.golang.org/grpc v1.28.1 // indirect diff --git a/learning/katas/go/Core Transforms/Map/ParDo Struct/go.sum b/learning/katas/go/Core Transforms/Map/ParDo Struct/go.sum index 66ee82f3d108..709c1101db07 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo Struct/go.sum +++ b/learning/katas/go/Core Transforms/Map/ParDo Struct/go.sum @@ -35,10 +35,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -82,6 +81,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -102,6 +102,7 @@ google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/learning/katas/go/Core Transforms/Map/ParDo Struct/task-remote-info.yaml b/learning/katas/go/Core Transforms/Map/ParDo Struct/task-remote-info.yaml index 190348ed6ee7..368c38dad01a 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo Struct/task-remote-info.yaml +++ b/learning/katas/go/Core Transforms/Map/ParDo Struct/task-remote-info.yaml @@ -1,2 +1,2 @@ id: 1289190 -update_date: Tue, 05 May 2020 02:01:59 UTC +update_date: Thu, 28 May 2020 22:33:45 UTC diff --git a/learning/katas/go/Core Transforms/Map/ParDo/go.mod b/learning/katas/go/Core Transforms/Map/ParDo/go.mod index da50fd0795d4..4393de855643 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo/go.mod +++ b/learning/katas/go/Core Transforms/Map/ParDo/go.mod @@ -15,12 +15,11 @@ module pardo -go 1.13 +go 1.14 require ( github.com/apache/beam v2.20.0+incompatible github.com/golang/protobuf v1.4.0 // indirect - github.com/googleapis/gax-go v2.0.2+incompatible // indirect go.opencensus.io v0.22.3 // indirect google.golang.org/api v0.21.0 // indirect google.golang.org/grpc v1.28.1 // indirect diff --git a/learning/katas/go/Core Transforms/Map/ParDo/go.sum b/learning/katas/go/Core Transforms/Map/ParDo/go.sum index 22b190304fca..395e4a1ba737 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo/go.sum +++ b/learning/katas/go/Core Transforms/Map/ParDo/go.sum @@ -35,10 +35,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -82,6 +81,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -102,6 +102,7 @@ google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY= google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/learning/katas/go/Core Transforms/Map/ParDo/task-remote-info.yaml b/learning/katas/go/Core Transforms/Map/ParDo/task-remote-info.yaml index 72b8745db6fc..42e50eda6d25 100644 --- a/learning/katas/go/Core Transforms/Map/ParDo/task-remote-info.yaml +++ b/learning/katas/go/Core Transforms/Map/ParDo/task-remote-info.yaml @@ -1,2 +1,2 @@ id: 1289188 -update_date: Tue, 05 May 2020 02:01:55 UTC +update_date: Thu, 28 May 2020 22:33:39 UTC diff --git a/learning/katas/go/Core Transforms/section-info.yaml b/learning/katas/go/Core Transforms/section-info.yaml index 23c69bef45f5..8bde6d156bf8 100644 --- a/learning/katas/go/Core Transforms/section-info.yaml +++ b/learning/katas/go/Core Transforms/section-info.yaml @@ -19,3 +19,7 @@ content: - Map +- GroupByKey +- CoGroupByKey +- Combine +- Flatten diff --git a/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/go.mod b/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/go.mod index 6cf809a0c9b5..696bd8f55c03 100644 --- a/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/go.mod +++ b/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/go.mod @@ -15,7 +15,7 @@ module hello_beam_test -go 1.13 +go 1.14 require ( github.com/apache/beam v2.19.0+incompatible diff --git a/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/task-remote-info.yaml b/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/task-remote-info.yaml index a75da50ff181..eaf8e5387bd1 100644 --- a/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/task-remote-info.yaml +++ b/learning/katas/go/Introduction/Hello Beam/Hello Beam Test/task-remote-info.yaml @@ -1,2 +1,2 @@ id: 1236832 -update_date: Tue, 05 May 2020 02:02:03 UTC +update_date: Thu, 28 May 2020 22:33:36 UTC diff --git a/learning/katas/go/Introduction/Hello Beam/Hello Beam/go.mod b/learning/katas/go/Introduction/Hello Beam/Hello Beam/go.mod index 13bb5997a52f..233fe5c13c7e 100644 --- a/learning/katas/go/Introduction/Hello Beam/Hello Beam/go.mod +++ b/learning/katas/go/Introduction/Hello Beam/Hello Beam/go.mod @@ -15,7 +15,7 @@ module hello_beam -go 1.13 +go 1.14 require ( github.com/apache/beam v2.19.0+incompatible diff --git a/learning/katas/go/Introduction/Hello Beam/Hello Beam/task-remote-info.yaml b/learning/katas/go/Introduction/Hello Beam/Hello Beam/task-remote-info.yaml index bf68a02eca35..7f138de415ba 100644 --- a/learning/katas/go/Introduction/Hello Beam/Hello Beam/task-remote-info.yaml +++ b/learning/katas/go/Introduction/Hello Beam/Hello Beam/task-remote-info.yaml @@ -1,2 +1,2 @@ id: 1236831 -update_date: Thu, 16 Apr 2020 06:53:05 UTC +update_date: Thu, 28 May 2020 22:33:33 UTC diff --git a/learning/katas/go/course-remote-info.yaml b/learning/katas/go/course-remote-info.yaml index 4252852155fc..ce7ead8cfc33 100644 --- a/learning/katas/go/course-remote-info.yaml +++ b/learning/katas/go/course-remote-info.yaml @@ -1,2 +1,2 @@ id: 70387 -update_date: Fri, 24 Apr 2020 07:39:05 UTC +update_date: Thu, 28 May 2020 22:34:30 UTC diff --git a/release/src/main/groovy/MobileGamingCommands.groovy b/release/src/main/groovy/MobileGamingCommands.groovy index 505fbcf691c0..1042062e4cb5 100644 --- a/release/src/main/groovy/MobileGamingCommands.groovy +++ b/release/src/main/groovy/MobileGamingCommands.groovy @@ -146,7 +146,7 @@ class MobileGamingCommands { } private Map getLeaderBoardWithStreamingEngineArgs(String runner, String jobName){ - args = [project: testScripts.gcpProject(), + def args = [project: testScripts.gcpProject(), dataset: testScripts.bqDataset(), topic: "projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}", leaderBoardTableName: "leaderboard_${runner}", @@ -160,7 +160,7 @@ class MobileGamingCommands { } private Map getGameStatsArgs(String runner, String jobName){ - args = [project: testScripts.gcpProject(), + def args = [project: testScripts.gcpProject(), dataset: testScripts.bqDataset(), topic: "projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}", fixedWindowDuration: 5, diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh index 36548d530f68..df5470bc2dec 100755 --- a/release/src/main/scripts/build_release_candidate.sh +++ b/release/src/main/scripts/build_release_candidate.sh @@ -113,7 +113,7 @@ if [[ $confirmation = "y" ]]; then echo "-------------Staging Java Artifacts into Maven---------------" gpg --local-user ${SIGNING_KEY} --output /dev/null --sign ~/.bashrc ./gradlew publish -Psigning.gnupg.keyName=${SIGNING_KEY} -PisRelease --no-daemon - echo "Please review all artifacts in staging URL. e.g. https://repository.apache.org/content/repositories/orgapachebeam-NNNN/" + echo "You need to close the staging repository manually on Apache Nexus. See the release guide for instructions." rm -rf ~/${LOCAL_CLONE_DIR} fi @@ -205,7 +205,7 @@ if [[ $confirmation = "y" ]]; then rm -rf ~/${PYTHON_ARTIFACTS_DIR} fi -echo "[Current Step]: Stage SDK docker images" +echo "[Current Step]: Stage docker images" echo "Do you want to proceed? [y|N]" read confirmation if [[ $confirmation = "y" ]]; then @@ -328,20 +328,3 @@ if [[ $confirmation = "y" ]]; then rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC} rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC} fi - -echo "===========Please Review All Items in the Checklist==========" -echo "1. Maven artifacts deployed to https://repository.apache.org/content/repositories/" -echo "2. Source distribution deployed to https://dist.apache.org/repos/dist/dev/beam/${RELEASE}" -echo "3. Website pull request published the Java API reference manual the Python API reference manual." - -echo "==============Things Needed To Be Done Manually==============" -echo "1.Make sure a pull request is created to update the javadoc and pydoc to the beam-site: " -echo " - cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_WEBSITE_REPO}/${WEBSITE_ROOT_DIR}" -echo " - git checkout updates_release_${RELEASE}" -echo " - Check if both javadoc/ and pydoc/ exist." -echo " - commit your changes" -echo "2.Create a pull request to update the release in the beam/website:" -echo " - An example pull request:https://github.com/apache/beam/pull/9341" -echo " - You can find the release note in JIRA: https://issues.apache.org/jira/projects/BEAM?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=unreleased" -echo "3.You need to build Python Wheels." -echo "4.Start the review-and-vote thread on the dev@ mailing list." diff --git a/release/src/main/scripts/cut_release_branch.sh b/release/src/main/scripts/cut_release_branch.sh index b3bdc832ea71..6d0f91379708 100755 --- a/release/src/main/scripts/cut_release_branch.sh +++ b/release/src/main/scripts/cut_release_branch.sh @@ -24,12 +24,12 @@ set -e function clean_up(){ - echo "Do you want to clean local clone repo? [y|N]" + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" read confirmation if [[ $confirmation = "y" ]]; then cd ~ rm -rf ${LOCAL_CLONE_DIR} - echo "Clean up local repo." + echo "Cleaned up local repo." fi } @@ -73,6 +73,7 @@ echo "===============================================================" cd ~ if [[ -d ${LOCAL_CLONE_DIR} ]]; then + echo "Deleting existing local clone repo ${LOCAL_CLONE_DIR}." rm -rf ${LOCAL_CLONE_DIR} fi mkdir ${LOCAL_CLONE_DIR} @@ -101,7 +102,7 @@ echo "===============================================================" echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]" read confirmation if [[ $confirmation != "y" ]]; then - echo "Exit without committing any changes on master branch." + echo "Exiting without committing any changes on master branch." clean_up exit fi @@ -136,7 +137,7 @@ echo "===============================================================" echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]" read confirmation if [[ $confirmation != "y" ]]; then - echo "Exit without committing any changes on release branch." + echo "Exiting without committing any changes on release branch." clean_up exit fi diff --git a/release/src/main/scripts/deploy_pypi.sh b/release/src/main/scripts/deploy_pypi.sh new file mode 100755 index 000000000000..daeb342a621e --- /dev/null +++ b/release/src/main/scripts/deploy_pypi.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script uploads Python artifacts staged at dist.apache.org to PyPI. + +set -e + +function clean_up(){ + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + cd ~ + rm -rf ${LOCAL_CLONE_DIR} + echo "Cleaned up local repo." + fi +} + +echo "Enter the release version, e.g. 2.21.0:" +read RELEASE +LOCAL_CLONE_DIR="beam_release_${RELEASE}" +cd ~ +if [[ -d ${LOCAL_CLONE_DIR} ]]; then + echo "Deleting existing local clone repo ${LOCAL_CLONE_DIR}." + rm -rf ${LOCAL_CLONE_DIR} +fi +mkdir ${LOCAL_CLONE_DIR} +cd ${LOCAL_CLONE_DIR} + +virtualenv deploy_pypi_env +source ./deploy_pypi_env/bin/activate +pip install twine + +wget -r --no-parent -A zip,whl "https://dist.apache.org/repos/dist/dev/beam/${RELEASE}/python" +cd "dist.apache.org/repos/dist/dev/beam/${RELEASE}/python/" +echo "Will upload the following files to PyPI:" +ls +echo "Are the files listed correct? [y|N]" +read confirmation +if [[ $confirmation != "y" ]]; then + echo "Exiting without deploying artifacts to PyPI." + clean_up + exit +fi +twine upload * + +clean_up diff --git a/release/src/main/scripts/mass_comment.py b/release/src/main/scripts/mass_comment.py new file mode 100644 index 000000000000..b0f74896dc22 --- /dev/null +++ b/release/src/main/scripts/mass_comment.py @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Script for mass-commenting Jenkins test triggers on a Beam PR.""" + +import itertools +import os +import socket +import sys +import time +import traceback +import re +import requests +from datetime import datetime + +COMMENTS_TO_ADD = [ + "Run Release Gradle Build", + "Run Go PostCommit", + "Run Java PostCommit", + "Run Java Flink PortableValidatesRunner Batch", + "Run Java Flink PortableValidatesRunner Streaming", + "Run Apex ValidatesRunner", + "Run Dataflow ValidatesRunner", + "Run Flink ValidatesRunner", + "Run Gearpump ValidatesRunner", + "Run Samza ValidatesRunner", + "Run Spark ValidatesRunner", + "Run Java Spark PortableValidatesRunner Batch", + "Run Python Spark ValidatesRunner", + "Run Python Dataflow ValidatesContainer", + "Run Python Dataflow ValidatesRunner", + "Run Python 3.5 Flink ValidatesRunner", + "Run Python 2 PostCommit", + "Run Python 3.5 PostCommit", + "Run Python 3.6 PostCommit", + "Run Python 3.7 PostCommit", + "Run SQL PostCommit", + "Run Go PreCommit", + "Run Java PreCommit", + "Run Java_Examples_Dataflow PreCommit", + "Run JavaPortabilityApi PreCommit", + "Run Portable_Python PreCommit", + "Run PythonLint PreCommit", + "Run Python PreCommit", + "Run Python DockerBuild PreCommit" +] + + +def executeGHGraphqlQuery(accessToken, query): + '''Runs graphql query on GitHub.''' + url = 'https://api.github.com/graphql' + headers = {'Authorization': 'Bearer %s' % accessToken} + r = requests.post(url=url, json={'query': query}, headers=headers) + return r.json() + + +def getSubjectId(accessToken, prNumber): + query = ''' +query FindPullRequestID { + repository(owner:"apache", name:"beam") { + pullRequest(number:%s) { + id + } + } +} +''' % prNumber + response = executeGHGraphqlQuery(accessToken, query) + return response['data']['repository']['pullRequest']['id'] + + +def fetchGHData(accessToken, subjectId, commentBody): + '''Fetches GitHub data required for reporting Beam metrics''' + query = ''' +mutation AddPullRequestComment { + addComment(input:{subjectId:"%s",body: "%s"}) { + commentEdge { + node { + createdAt + body + } + } + subject { + id + } + } +} +''' % (subjectId, commentBody) + return executeGHGraphqlQuery(accessToken, query) + + +def postComments(accessToken, subjectId): + ''' + Main workhorse method. Fetches data from GitHub and puts it in metrics table. + ''' + + for commentBody in COMMENTS_TO_ADD: + jsonData = fetchGHData(accessToken, subjectId, commentBody) + print(jsonData) + + +def probeGitHubIsUp(): + ''' + Returns True if GitHub responds to simple queries. Else returns False. + ''' + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(('github.com', 443)) + return True if result == 0 else False + + +################################################################################ +if __name__ == '__main__': + ''' + This script is supposed to be invoked directly. + However for testing purposes and to allow importing, + wrap work code in module check. + ''' + print("Started.") + + if not probeGitHubIsUp(): + print("GitHub is unavailable, skipping fetching data.") + exit() + + print("GitHub is available start fetching data.") + + accessToken = input("Enter your Github access token: ") + + pr = input("Enter the Beam PR number to test (e.g. 11403): ") + subjectId = getSubjectId(accessToken, pr) + + postComments(accessToken, subjectId) + print("Fetched data.") + + print('Done.') diff --git a/release/src/main/scripts/publish_docker_images.sh b/release/src/main/scripts/publish_docker_images.sh index 1fb646313861..7575107ebf11 100755 --- a/release/src/main/scripts/publish_docker_images.sh +++ b/release/src/main/scripts/publish_docker_images.sh @@ -28,7 +28,7 @@ DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_ PYTHON_VER=("python2.7" "python3.5" "python3.6" "python3.7") -FLINK_VER=("1.7" "1.8" "1.9") +FLINK_VER=("1.8" "1.9" "1.10") echo "Publish SDK docker images to Docker Hub." diff --git a/release/src/main/scripts/verify_release_build.sh b/release/src/main/scripts/verify_release_build.sh index bc3671f46d93..77c184bc463a 100755 --- a/release/src/main/scripts/verify_release_build.sh +++ b/release/src/main/scripts/verify_release_build.sh @@ -40,40 +40,6 @@ BEAM_REPO_URL=https://github.com/apache/beam.git RELEASE_BRANCH=release-${RELEASE_VER} WORKING_BRANCH=postcommit_validation_pr -JOB_TRIGGER_PHRASES=( - # To verify Gradle release build - "**Run Release Gradle Build**" - # To run all PostCommit jobs - "Run Go PostCommit" - "Run Java PostCommit" - "Run Java PortabilityApi PostCommit" - "Run Java Flink PortableValidatesRunner Batch" - "Run Java Flink PortableValidatesRunner Streaming" - "Run Apex ValidatesRunner" - "Run Dataflow ValidatesRunner" - "Run Flink ValidatesRunner" - "Run Gearpump ValidatesRunner" - "Run Dataflow PortabilityApi ValidatesRunner" - "Run Samza ValidatesRunner" - "Run Spark ValidatesRunner" - "Run Python Dataflow ValidatesContainer" - "Run Python Dataflow ValidatesRunner" - "Run Python 3.5 Flink ValidatesRunner" - # Python versions match those in run_rc_validation.sh. - "Run Python 2 PostCommit" - "Run Python 3.5 PostCommit" - "Run SQL PostCommit" - "Run Go PreCommit" - "Run Java PreCommit" - "Run Java_Examples_Dataflow PreCommit" - "Run JavaPortabilityApi PreCommit" - "Run Portable_Python PreCommit" - "Run PythonLint PreCommit" - "Run Python PreCommit" - "Run Python DockerBuild PreCommit" -) - - function clean_up(){ echo "" echo "==================== Final Cleanup ====================" @@ -164,12 +130,9 @@ if [[ ! -z `which hub` ]]; then git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push -f ${GITHUB_USERNAME} --quiet - trigger_phrases=$(IFS=$'\n'; echo "${JOB_TRIGGER_PHRASES[*]}") hub pull-request -b apache:${RELEASE_BRANCH} -h ${GITHUB_USERNAME}:${WORKING_BRANCH} -F- <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch - Please comment as instructions below, one phrase per comment: - - ${trigger_phrases}" + You can run many tests automatically using release/src/main/scripts/mass_comment.py." echo "" echo "[NOTE]: Please make sure all test targets have been invoked." diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java index 564a8d3db3cc..53a566c1b52c 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import org.apache.avro.Schema; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.AvroGenericCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; @@ -37,7 +38,8 @@ public byte[] getPayload(AvroGenericCoder from) { } @Override - public AvroGenericCoder fromComponents(List> components, byte[] payload) { + public AvroGenericCoder fromComponents( + List> components, byte[] payload, TranslationContext context) { Schema schema = new Schema.Parser().parse(new String(payload, Charsets.UTF_8)); return AvroGenericCoder.of(schema); } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java index 86f017814d53..6f6c59b20df9 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java @@ -36,6 +36,22 @@ /** Converts to and from Beam Runner API representations of {@link Coder Coders}. */ public class CoderTranslation { + + /** + * Pass through additional parameters beyond the components and payload to be able to translate + * specific coders. + * + *

Portability state API backed coders is an example of such a coder translator requiring + * additional parameters. + */ + public interface TranslationContext { + /** The default translation context containing no additional parameters. */ + TranslationContext DEFAULT = new DefaultTranslationContext(); + } + + /** A convenient class representing a default context containing no additional parameters. */ + private static class DefaultTranslationContext implements TranslationContext {} + // This URN says that the coder is just a UDF blob this SDK understands // TODO: standardize such things public static final String JAVA_SERIALIZED_CODER_URN = "beam:coders:javasdk:0.1"; @@ -115,21 +131,29 @@ private static RunnerApi.Coder toCustomCoder(Coder coder) throws IOException .build(); } - public static Coder fromProto(RunnerApi.Coder protoCoder, RehydratedComponents components) + public static Coder fromProto( + RunnerApi.Coder protoCoder, RehydratedComponents components, TranslationContext context) throws IOException { String coderSpecUrn = protoCoder.getSpec().getUrn(); if (coderSpecUrn.equals(JAVA_SERIALIZED_CODER_URN)) { return fromCustomCoder(protoCoder); } - return fromKnownCoder(protoCoder, components); + return fromKnownCoder(protoCoder, components, context); } - private static Coder fromKnownCoder(RunnerApi.Coder coder, RehydratedComponents components) + private static Coder fromKnownCoder( + RunnerApi.Coder coder, RehydratedComponents components, TranslationContext context) throws IOException { String coderUrn = coder.getSpec().getUrn(); List> coderComponents = new ArrayList<>(); for (String componentId : coder.getComponentCoderIdsList()) { - Coder innerCoder = components.getCoder(componentId); + // Only store coders in RehydratedComponents as long as we are not using a custom + // translation context. + Coder innerCoder = + context == TranslationContext.DEFAULT + ? components.getCoder(componentId) + : fromProto( + components.getComponents().getCodersOrThrow(componentId), components, context); coderComponents.add(innerCoder); } Class coderType = KNOWN_CODER_URNS.inverse().get(coderUrn); @@ -139,7 +163,8 @@ private static Coder fromKnownCoder(RunnerApi.Coder coder, RehydratedComponen "Unknown Coder URN %s. Known URNs: %s", coderUrn, KNOWN_CODER_URNS.values()); - return translator.fromComponents(coderComponents, coder.getSpec().getPayload().toByteArray()); + return translator.fromComponents( + coderComponents, coder.getSpec().getPayload().toByteArray(), context); } private static Coder fromCustomCoder(RunnerApi.Coder protoCoder) throws IOException { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslator.java index 442a445df355..434ac1b0957b 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslator.java @@ -18,6 +18,7 @@ package org.apache.beam.runners.core.construction; import java.util.List; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.Coder; /** @@ -41,6 +42,9 @@ default byte[] getPayload(T from) { return new byte[0]; } - /** Create a {@link Coder} from its component {@link Coder coders}. */ - T fromComponents(List> components, byte[] payload); + /** + * Create a {@link Coder} from its component {@link Coder coders} using the specified translation + * context. + */ + T fromComponents(List> components, byte[] payload, TranslationContext context); } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java index ae7e9dc4b425..c4f45023d912 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import org.apache.beam.model.pipeline.v1.SchemaApi; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.IterableCoder; import org.apache.beam.sdk.coders.KvCoder; @@ -139,7 +140,7 @@ public byte[] getPayload(WindowedValue.ParamWindowedValueCoder from) { @Override public WindowedValue.ParamWindowedValueCoder fromComponents( - List> components, byte[] payload) { + List> components, byte[] payload, TranslationContext context) { return WindowedValue.ParamWindowedValueCoder.fromComponents(components, payload); } }; @@ -158,7 +159,8 @@ public byte[] getPayload(RowCoder from) { } @Override - public RowCoder fromComponents(List> components, byte[] payload) { + public RowCoder fromComponents( + List> components, byte[] payload, TranslationContext context) { checkArgument( components.isEmpty(), "Expected empty component list, but received: " + components); Schema schema; @@ -175,7 +177,8 @@ public RowCoder fromComponents(List> components, byte[] payload) { public abstract static class SimpleStructuredCoderTranslator> implements CoderTranslator { @Override - public final T fromComponents(List> components, byte[] payload) { + public final T fromComponents( + List> components, byte[] payload, TranslationContext context) { return fromComponents(components); } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java index c324b92e7f33..72ebd13b681c 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java @@ -17,6 +17,7 @@ */ package org.apache.beam.runners.core.construction; +import com.fasterxml.jackson.core.Base64Variants; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.FileOutputStream; @@ -215,25 +216,19 @@ public static List getArtifacts(List stagingFiles) Set deduplicatedStagingFiles = new LinkedHashSet<>(stagingFiles); for (String path : deduplicatedStagingFiles) { File file; - String stagedName; + String stagedName = null; if (path.contains("=")) { String[] components = path.split("=", 2); file = new File(components[1]); stagedName = components[0]; } else { file = new File(path); - stagedName = createStagingFileName(file); } // Spurious items get added to the classpath. Filter by just those that exist. if (file.exists()) { ArtifactInformation.Builder artifactBuilder = ArtifactInformation.newBuilder(); artifactBuilder.setTypeUrn(BeamUrns.getUrn(StandardArtifacts.Types.FILE)); artifactBuilder.setRoleUrn(BeamUrns.getUrn(StandardArtifacts.Roles.STAGING_TO)); - artifactBuilder.setRolePayload( - RunnerApi.ArtifactStagingToRolePayload.newBuilder() - .setStagedName(stagedName) - .build() - .toByteString()); HashCode hashCode; if (file.isDirectory()) { File zippedFile; @@ -264,6 +259,14 @@ public static List getArtifacts(List stagingFiles) .build() .toByteString()); } + if (stagedName == null) { + stagedName = createStagingFileName(file, hashCode); + } + artifactBuilder.setRolePayload( + RunnerApi.ArtifactStagingToRolePayload.newBuilder() + .setStagedName(stagedName) + .build() + .toByteString()); artifactsBuilder.add(artifactBuilder.build()); } } @@ -314,10 +317,12 @@ public static Set getJavaCapabilities() { return capabilities.build(); } - public static String createStagingFileName(File path) { + public static String createStagingFileName(File path, HashCode hash) { + String encodedHash = Base64Variants.MODIFIED_FOR_URL.encode(hash.asBytes()); + String fileName = Files.getNameWithoutExtension(path.getAbsolutePath()); String ext = path.isDirectory() ? "jar" : Files.getFileExtension(path.getAbsolutePath()); String suffix = Strings.isNullOrEmpty(ext) ? "" : "." + ext; - return UUID.randomUUID().toString() + suffix; + return String.format("%s-%s%s", fileName, encodedHash, suffix); } private static File zipDirectory(File directory) throws IOException { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java index 8ff90630f396..501a82379506 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java @@ -20,6 +20,7 @@ import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; +import java.util.Collections; import java.util.Map; import java.util.Set; import org.apache.beam.sdk.coders.BooleanCoder; @@ -97,7 +98,15 @@ public class ModelCoderRegistrar implements CoderTranslatorRegistrar { CoderTranslator.class.getSimpleName(), Sets.difference(BEAM_MODEL_CODER_URNS.keySet(), BEAM_MODEL_CODERS.keySet())); checkState( - ModelCoders.urns().equals(BEAM_MODEL_CODER_URNS.values()), + Sets.symmetricDifference( + ModelCoders.urns(), + /** + * The state backed iterable coder implementation is environment specific and hence + * is not part of the coder translation checks as these are meant to be used only + * during pipeline construction. + */ + Collections.singleton(ModelCoders.STATE_BACKED_ITERABLE_CODER_URN)) + .equals(BEAM_MODEL_CODER_URNS.values()), "All Model %ss should have an associated java %s", Coder.class.getSimpleName(), Coder.class.getSimpleName()); diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java index 7685b730510b..e1c204748bb4 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java @@ -19,6 +19,7 @@ import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Set; @@ -59,6 +60,14 @@ private ModelCoders() {} public static final String ROW_CODER_URN = getUrn(StandardCoders.Enum.ROW); + public static final String STATE_BACKED_ITERABLE_CODER_URN = + "beam:coder:state_backed_iterable:v1"; + + static { + checkState( + STATE_BACKED_ITERABLE_CODER_URN.equals(getUrn(StandardCoders.Enum.STATE_BACKED_ITERABLE))); + } + private static final Set MODEL_CODER_URNS = ImmutableSet.of( BYTES_CODER_URN, @@ -74,7 +83,8 @@ private ModelCoders() {} WINDOWED_VALUE_CODER_URN, DOUBLE_CODER_URN, ROW_CODER_URN, - PARAM_WINDOWED_VALUE_CODER_URN); + PARAM_WINDOWED_VALUE_CODER_URN, + STATE_BACKED_ITERABLE_CODER_URN); public static Set urns() { return MODEL_CODER_URNS; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/NativeTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/NativeTransforms.java index 730a5ca561f5..9ea47a10835b 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/NativeTransforms.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/NativeTransforms.java @@ -46,7 +46,10 @@ public class NativeTransforms { * Returns true if an only if the Runner understands this transform and can handle it directly. */ public static boolean isNative(RunnerApi.PTransform pTransform) { - Iterator matchers = ServiceLoader.load(IsNativeTransform.class).iterator(); + // TODO(BEAM-10109) Use default (context) classloader. + Iterator matchers = + ServiceLoader.load(IsNativeTransform.class, NativeTransforms.class.getClassLoader()) + .iterator(); while (matchers.hasNext()) { if (matchers.next().test(pTransform)) { return true; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java index 11d24042c626..c51508b14bfd 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java @@ -27,6 +27,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.Components; import org.apache.beam.model.pipeline.v1.RunnerApi.Environment; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.values.PCollection; @@ -81,7 +82,8 @@ public class RehydratedComponents { public Coder load(String id) throws Exception { @Nullable RunnerApi.Coder coder = components.getCodersOrDefault(id, null); checkState(coder != null, "No coder with id '%s' in serialized components", id); - return CoderTranslation.fromProto(coder, RehydratedComponents.this); + return CoderTranslation.fromProto( + coder, RehydratedComponents.this, TranslationContext.DEFAULT); } }); diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java index e6b45c475d52..8bc677214c86 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java @@ -31,6 +31,7 @@ import org.apache.avro.SchemaBuilder; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.Components; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.coders.AvroCoder; import org.apache.beam.sdk.coders.BooleanCoder; @@ -165,7 +166,9 @@ public void toAndFromProto() throws Exception { Components encodedComponents = sdkComponents.toComponents(); Coder decodedCoder = CoderTranslation.fromProto( - coderProto, RehydratedComponents.forComponents(encodedComponents)); + coderProto, + RehydratedComponents.forComponents(encodedComponents), + TranslationContext.DEFAULT); assertThat(decodedCoder, equalTo(coder)); if (KNOWN_CODERS.contains(coder)) { diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CommonCoderTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CommonCoderTest.java index fe08582b9bbb..7b0d5453f829 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CommonCoderTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CommonCoderTest.java @@ -49,6 +49,7 @@ import javax.annotation.Nullable; import org.apache.beam.model.pipeline.v1.RunnerApi.StandardCoders; import org.apache.beam.model.pipeline.v1.SchemaApi; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.BooleanCoder; import org.apache.beam.sdk.coders.ByteCoder; import org.apache.beam.sdk.coders.Coder; @@ -407,7 +408,7 @@ private static Coder instantiateCoder(CommonCoder coder) { checkNotNull( translator, "No translator found for common coder class: " + coderType.getSimpleName()); - return translator.fromComponents(components, coder.getPayload()); + return translator.fromComponents(components, coder.getPayload(), new TranslationContext() {}); } @Test diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java index 113186eafc7f..5c3b388447fa 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java @@ -29,6 +29,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload; import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.KvCoder; @@ -200,7 +201,8 @@ public void toTransformProto() throws Exception { Coder timerCoder = CoderTranslation.fromProto( components.getCodersOrThrow(timerFamilySpec.getTimerFamilyCoderId()), - rehydratedComponents); + rehydratedComponents, + TranslationContext.DEFAULT); assertEquals( org.apache.beam.runners.core.construction.Timer.Coder.of( VarLongCoder.of(), GlobalWindow.Coder.INSTANCE), diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java index 9cffd69ab3ac..63264a6897f3 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java @@ -29,6 +29,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.CombinePayload; import org.apache.beam.model.pipeline.v1.RunnerApi.Components; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.Pipeline.PipelineVisitor; import org.apache.beam.sdk.coders.BigEndianLongCoder; @@ -213,7 +214,9 @@ private static Coder getAccumulatorCoder(AppliedPTransform transform .orElseThrow(() -> new IOException("Transform does not contain an AccumulatorCoder")); Components components = sdkComponents.toComponents(); return CoderTranslation.fromProto( - components.getCodersOrThrow(id), RehydratedComponents.forComponents(components)); + components.getCodersOrThrow(id), + RehydratedComponents.forComponents(components), + TranslationContext.DEFAULT); } private static Optional getCombinePayload( diff --git a/runners/flink/1.10/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java b/runners/flink/1.10/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java new file mode 100644 index 000000000000..1b56c72946a9 --- /dev/null +++ b/runners/flink/1.10/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink; + +/** Handle different capabilities between flink versions. */ +public class FlinkCapabilities { + + /** + * Support for outputting elements in close method of chained drivers. + * + *

{@see FLINK-14709} for more + * details. + * + * @return True if feature is supported. + */ + public static boolean supportsOutputDuringClosing() { + return true; + } +} diff --git a/runners/flink/1.8/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java b/runners/flink/1.8/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java new file mode 100644 index 000000000000..e1d2a44f9597 --- /dev/null +++ b/runners/flink/1.8/src/main/java/org/apache/beam/runners/flink/FlinkCapabilities.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink; + +/** Handle different capabilities between flink versions. */ +public class FlinkCapabilities { + + /** + * Support for outputting elements in close method of chained drivers. + * + *

{@see FLINK-14709} for more + * details. + * + * @return True if feature is supported. + */ + public static boolean supportsOutputDuringClosing() { + return false; + } +} diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index ed76e6b89365..0ce92a569dc0 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -163,6 +163,10 @@ def portableValidatesRunnerTask(String name, Boolean streaming) { excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs' excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' }, + testFilter: { + // TODO(BEAM-10016) + excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' + }, ) } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java index 772c9c9a8dce..7286713019ef 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java @@ -46,6 +46,7 @@ public FlinkBatchPipelineTranslator(ExecutionEnvironment env, PipelineOptions op @Override @SuppressWarnings("rawtypes, unchecked") public void translate(Pipeline pipeline) { + batchContext.init(pipeline); super.translate(pipeline); // terminate dangling DataSets @@ -63,7 +64,7 @@ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) { LOG.info("{} enterCompositeTransform- {}", genSpaces(this.depth), node.getFullName()); this.depth++; - BatchTransformTranslator translator = getTranslator(node); + BatchTransformTranslator translator = getTranslator(node, batchContext); if (translator != null) { applyBatchTransform(node.getTransform(), node, translator); @@ -88,7 +89,7 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) { // currently visiting and translate it into its Flink alternative. PTransform transform = node.getTransform(); BatchTransformTranslator translator = - FlinkBatchTransformTranslators.getTranslator(transform); + FlinkBatchTransformTranslators.getTranslator(transform, batchContext); if (translator == null) { String transformUrn = PTransformTranslation.urnForTransform(transform); throw new UnsupportedOperationException( @@ -115,11 +116,17 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) { /** A translator of a {@link PTransform}. */ public interface BatchTransformTranslator { + + default boolean canTranslate(TransformT transform, FlinkBatchTranslationContext context) { + return true; + } + void translateNode(TransformT transform, FlinkBatchTranslationContext context); } /** Returns a translator for the given node, if it is possible, otherwise null. */ - private static BatchTransformTranslator getTranslator(TransformHierarchy.Node node) { + private static BatchTransformTranslator getTranslator( + TransformHierarchy.Node node, FlinkBatchTranslationContext context) { @Nullable PTransform transform = node.getTransform(); // Root of the graph is null @@ -127,6 +134,6 @@ private static BatchTransformTranslator getTranslator(TransformHierarchy.Node return null; } - return FlinkBatchTransformTranslators.getTranslator(transform); + return FlinkBatchTransformTranslators.getTranslator(transform, context); } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java index 4254a0604c78..264fa4c43302 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java @@ -34,19 +34,23 @@ import org.apache.beam.runners.core.construction.ReadTranslation; import org.apache.beam.runners.flink.translation.functions.FlinkAssignWindows; import org.apache.beam.runners.flink.translation.functions.FlinkDoFnFunction; +import org.apache.beam.runners.flink.translation.functions.FlinkExplodeWindowsFunction; import org.apache.beam.runners.flink.translation.functions.FlinkIdentityFunction; import org.apache.beam.runners.flink.translation.functions.FlinkMergingNonShuffleReduceFunction; import org.apache.beam.runners.flink.translation.functions.FlinkMultiOutputPruningFunction; +import org.apache.beam.runners.flink.translation.functions.FlinkNonMergingReduceFunction; import org.apache.beam.runners.flink.translation.functions.FlinkPartialReduceFunction; import org.apache.beam.runners.flink.translation.functions.FlinkReduceFunction; import org.apache.beam.runners.flink.translation.functions.FlinkStatefulDoFnFunction; import org.apache.beam.runners.flink.translation.types.CoderTypeInformation; import org.apache.beam.runners.flink.translation.types.KvKeySelector; +import org.apache.beam.runners.flink.translation.types.WindowedKvKeySelector; import org.apache.beam.runners.flink.translation.wrappers.ImpulseInputFormat; import org.apache.beam.runners.flink.translation.wrappers.SourceInputFormat; import org.apache.beam.sdk.coders.CannotProvideCoderException; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderRegistry; +import org.apache.beam.sdk.coders.IterableCoder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.VoidCoder; @@ -65,6 +69,7 @@ import org.apache.beam.sdk.transforms.reflect.DoFnSignatures; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; +import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; @@ -78,6 +83,8 @@ import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.MultimapBuilder; import org.apache.flink.api.common.functions.RichGroupReduceFunction; import org.apache.flink.api.common.operators.Order; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -91,6 +98,7 @@ import org.apache.flink.api.java.operators.MapOperator; import org.apache.flink.api.java.operators.MapPartitionOperator; import org.apache.flink.api.java.operators.SingleInputUdfOperator; +import org.apache.flink.api.java.operators.UnsortedGrouping; import org.apache.flink.configuration.Configuration; import org.apache.flink.optimizer.Optimizer; import org.joda.time.Instant; @@ -105,40 +113,45 @@ class FlinkBatchTransformTranslators { // -------------------------------------------------------------------------------------------- @SuppressWarnings("rawtypes") - private static final Map - TRANSLATORS = new HashMap<>(); + private static final Multimap + TRANSLATORS = MultimapBuilder.hashKeys().arrayListValues().build(); static { TRANSLATORS.put(PTransformTranslation.IMPULSE_TRANSFORM_URN, new ImpulseTranslatorBatch()); - TRANSLATORS.put( PTransformTranslation.CREATE_VIEW_TRANSFORM_URN, - new CreatePCollectionViewTranslatorBatch()); - + new CreatePCollectionViewTranslatorBatch<>()); TRANSLATORS.put( - PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch()); + PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch<>()); TRANSLATORS.put( - PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch()); - TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch()); - + PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, + new NonMergingGroupByKeyTranslatorBatch<>()); TRANSLATORS.put( - PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch()); - + PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch<>()); + TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch<>()); TRANSLATORS.put( - PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch()); - - TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch()); - - TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch()); + PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch<>()); + TRANSLATORS.put( + PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch<>()); + TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch<>()); + TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch<>()); } + @SuppressWarnings("unchecked") static FlinkBatchPipelineTranslator.BatchTransformTranslator getTranslator( - PTransform transform) { - @Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform); - return urn == null ? null : TRANSLATORS.get(urn); + PTransform transform, FlinkBatchTranslationContext context) { + @Nullable final String urn = PTransformTranslation.urnForTransformOrNull(transform); + if (urn != null && TRANSLATORS.containsKey(urn)) { + for (FlinkBatchPipelineTranslator.BatchTransformTranslator> translator : + TRANSLATORS.get(urn)) { + if (translator.canTranslate(transform, context)) { + return translator; + } + } + } + return null; } - @SuppressWarnings("unchecked") private static String getCurrentTransformName(FlinkBatchTranslationContext context) { return context.getCurrentTransform().getFullName(); } @@ -231,6 +244,73 @@ public void translateNode( } } + /** + * Optimized group by key translation for non-merging windows. + * + * @param Key type. + * @param Input type. + */ + private static class NonMergingGroupByKeyTranslatorBatch + implements FlinkBatchPipelineTranslator.BatchTransformTranslator< + PTransform>, PCollection>>>> { + + @Override + public boolean canTranslate( + PTransform>, PCollection>>> transform, + FlinkBatchTranslationContext context) { + final WindowingStrategy windowingStrategy = + context.getInput(transform).getWindowingStrategy(); + return windowingStrategy.getWindowFn().isNonMerging() + && windowingStrategy.getTimestampCombiner() == TimestampCombiner.END_OF_WINDOW + && windowingStrategy.getWindowFn().windowCoder().consistentWithEquals(); + } + + @Override + public void translateNode( + PTransform>, PCollection>>> transform, + FlinkBatchTranslationContext context) { + final int numConsumers = + context.getOutputs(transform).values().stream().mapToInt(context::getNumConsumers).sum(); + final boolean multipleConsumers = numConsumers > 1; + final boolean reIterableResult = + multipleConsumers + || context + .getPipelineOptions() + .as(FlinkPipelineOptions.class) + .getReIterableGroupByKeyResult(); + final DataSet>> inputDataSet = + context.getInputDataSet(context.getInput(transform)); + final KvCoder inputCoder = + (KvCoder) context.getInput(transform).getCoder(); + final WindowingStrategy windowingStrategy = + context.getInput(transform).getWindowingStrategy(); + final String fullName = getCurrentTransformName(context); + final UnsortedGrouping>> inputGrouping = + new FlatMapOperator<>( + inputDataSet, + inputDataSet.getType(), + new FlinkExplodeWindowsFunction<>(), + "ExplodeWindows: " + fullName) + .groupBy( + new WindowedKvKeySelector<>( + inputCoder.getKeyCoder(), windowingStrategy.getWindowFn().windowCoder())); + final TypeInformation>>> outputTypeInfo = + new CoderTypeInformation<>( + WindowedValue.getFullCoder( + KvCoder.of( + inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getValueCoder())), + windowingStrategy.getWindowFn().windowCoder())); + final DataSet>>> outputDataSet = + new GroupReduceOperator<>( + inputGrouping, + outputTypeInfo, + new FlinkNonMergingReduceFunction<>(windowingStrategy, reIterableResult), + fullName) + .returns(outputTypeInfo); + context.setOutputDataSet(context.getOutput(transform), outputDataSet); + } + } + private static class GroupByKeyTranslatorBatch implements FlinkBatchPipelineTranslator.BatchTransformTranslator< PTransform>, PCollection>>>> { @@ -240,7 +320,7 @@ public void translateNode( PTransform>, PCollection>>> transform, FlinkBatchTranslationContext context) { - // for now, this is copied from the Combine.PerKey translater. Once we have the new runner API + // for now, this is copied from the Combine.PerKey translator. Once we have the new runner API // we can replace GroupByKey by a Combine.PerKey with the Concatenate CombineFn DataSet>> inputDataSet = @@ -527,11 +607,12 @@ public void translateNode( Map, PValue> outputs = context.getOutputs(transform); - TupleTag mainOutputTag; + final TupleTag mainOutputTag; DoFnSchemaInformation doFnSchemaInformation; Map> sideInputMapping; try { - mainOutputTag = ParDoTranslation.getMainOutputTag(context.getCurrentTransform()); + mainOutputTag = + (TupleTag) ParDoTranslation.getMainOutputTag(context.getCurrentTransform()); } catch (IOException e) { throw new RuntimeException(e); } @@ -603,7 +684,7 @@ public void translateNode( throw new RuntimeException(e); } - Map, Coder> outputCoderMap = context.getOutputCoders(); + final Map, Coder> outputCoderMap = context.getOutputCoders(transform); String fullName = getCurrentTransformName(context); if (usesStateOrTimers) { @@ -638,8 +719,8 @@ public void translateNode( outputDataSet = new GroupReduceOperator(grouping, typeInformation, doFnWrapper, fullName); } else { - FlinkDoFnFunction doFnWrapper = - new FlinkDoFnFunction( + final FlinkDoFnFunction doFnWrapper = + new FlinkDoFnFunction<>( doFn, fullName, windowingStrategy, @@ -652,8 +733,14 @@ public void translateNode( doFnSchemaInformation, sideInputMapping); - outputDataSet = - new MapPartitionOperator<>(inputDataSet, typeInformation, doFnWrapper, fullName); + if (FlinkCapabilities.supportsOutputDuringClosing()) { + outputDataSet = + new FlatMapOperator<>(inputDataSet, typeInformation, doFnWrapper, fullName); + } else { + // This can be removed once we drop support for 1.8 and 1.9 versions. + outputDataSet = + new MapPartitionOperator<>(inputDataSet, typeInformation, doFnWrapper, fullName); + } } transformSideInputs(sideInputs, outputDataSet, context); @@ -789,6 +876,4 @@ private static void transformSideInputs( outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } } - - private FlinkBatchTransformTranslators() {} } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java index 8e366cf85d42..1af8020cdc19 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java @@ -19,9 +19,10 @@ import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; -import org.apache.beam.runners.core.construction.TransformInputs; import org.apache.beam.runners.flink.translation.types.CoderTypeInformation; +import org.apache.beam.runners.flink.translation.utils.CountingPipelineVisitor; +import org.apache.beam.runners.flink.translation.utils.LookupPipelineVisitor; +import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.runners.AppliedPTransform; @@ -32,7 +33,6 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; @@ -57,9 +57,12 @@ class FlinkBatchTranslationContext { private AppliedPTransform currentTransform; + private final CountingPipelineVisitor countingPipelineVisitor = new CountingPipelineVisitor(); + private final LookupPipelineVisitor lookupPipelineVisitor = new LookupPipelineVisitor(); + // ------------------------------------------------------------------------ - public FlinkBatchTranslationContext(ExecutionEnvironment env, PipelineOptions options) { + FlinkBatchTranslationContext(ExecutionEnvironment env, PipelineOptions options) { this.env = env; this.options = options; this.dataSets = new HashMap<>(); @@ -68,13 +71,18 @@ public FlinkBatchTranslationContext(ExecutionEnvironment env, PipelineOptions op this.danglingDataSets = new HashMap<>(); } + void init(Pipeline pipeline) { + pipeline.traverseTopologically(countingPipelineVisitor); + pipeline.traverseTopologically(lookupPipelineVisitor); + } + // ------------------------------------------------------------------------ - public Map> getDanglingDataSets() { + Map> getDanglingDataSets() { return danglingDataSets; } - public ExecutionEnvironment getExecutionEnvironment() { + ExecutionEnvironment getExecutionEnvironment() { return env; } @@ -83,13 +91,13 @@ public PipelineOptions getPipelineOptions() { } @SuppressWarnings("unchecked") - public DataSet> getInputDataSet(PValue value) { + DataSet> getInputDataSet(PValue value) { // assume that the DataSet is used as an input if retrieved here danglingDataSets.remove(value); return (DataSet>) dataSets.get(value); } - public void setOutputDataSet(PValue value, DataSet> set) { + void setOutputDataSet(PValue value, DataSet> set) { if (!dataSets.containsKey(value)) { dataSets.put(value, set); danglingDataSets.put(value, set); @@ -99,41 +107,37 @@ public void setOutputDataSet(PValue value, DataSet> set) { /** * Sets the AppliedPTransform which carries input/output. * - * @param currentTransform + * @param currentTransform Current transformation. */ - public void setCurrentTransform(AppliedPTransform currentTransform) { + void setCurrentTransform(AppliedPTransform currentTransform) { this.currentTransform = currentTransform; } - public AppliedPTransform getCurrentTransform() { + AppliedPTransform getCurrentTransform() { return currentTransform; } - public Map, Coder> getOutputCoders() { - return currentTransform.getOutputs().entrySet().stream() - .filter(e -> e.getValue() instanceof PCollection) - .collect(Collectors.toMap(e -> e.getKey(), e -> ((PCollection) e.getValue()).getCoder())); + Map, Coder> getOutputCoders(PTransform transform) { + return lookupPipelineVisitor.getOutputCoders(transform); } @SuppressWarnings("unchecked") - public DataSet getSideInputDataSet(PCollectionView value) { + DataSet getSideInputDataSet(PCollectionView value) { return (DataSet) broadcastDataSets.get(value); } - public void setSideInputDataSet( + void setSideInputDataSet( PCollectionView value, DataSet> set) { if (!broadcastDataSets.containsKey(value)) { broadcastDataSets.put(value, set); } } - @SuppressWarnings("unchecked") - public TypeInformation> getTypeInfo(PCollection collection) { + TypeInformation> getTypeInfo(PCollection collection) { return getTypeInfo(collection.getCoder(), collection.getWindowingStrategy()); } - @SuppressWarnings("unchecked") - public TypeInformation> getTypeInfo( + TypeInformation> getTypeInfo( Coder coder, WindowingStrategy windowingStrategy) { WindowedValue.FullWindowedValueCoder windowedValueCoder = WindowedValue.getFullCoder(coder, windowingStrategy.getWindowFn().windowCoder()); @@ -142,20 +146,23 @@ public TypeInformation> getTypeInfo( } Map, PValue> getInputs(PTransform transform) { - return currentTransform.getInputs(); + return lookupPipelineVisitor.getInputs(transform); } - @SuppressWarnings("unchecked") T getInput(PTransform transform) { - return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); + return lookupPipelineVisitor.getInput(transform); } Map, PValue> getOutputs(PTransform transform) { - return currentTransform.getOutputs(); + return lookupPipelineVisitor.getOutputs(transform); } - @SuppressWarnings("unchecked") T getOutput(PTransform transform) { - return (T) Iterables.getOnlyElement(currentTransform.getOutputs().values()); + return lookupPipelineVisitor.getOutput(transform); + } + + /** {@link CountingPipelineVisitor#getNumConsumers(PValue)}. */ + int getNumConsumers(PValue value) { + return countingPipelineVisitor.getNumConsumers(value); } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java index 22bff18dbdc2..6c42e2f5efb4 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java @@ -267,4 +267,11 @@ public interface FlinkPipelineOptions String getReportCheckpointDuration(); void setReportCheckpointDuration(String metricNamespace); + + @Description( + "Flag indicating whether result of GBK needs to be re-iterable. Re-iterable result implies that all values for a single key must fit in memory as we currently do not support spilling to disk.") + @Default.Boolean(false) + Boolean getReIterableGroupByKeyResult(); + + void setReIterableGroupByKeyResult(Boolean reIterableGroupByKeyResult); } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java index a34d840e7417..7a77c8d8ddfe 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java @@ -19,7 +19,9 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import javax.annotation.Nullable; import org.apache.beam.runners.core.DoFnRunner; import org.apache.beam.runners.core.DoFnRunners; import org.apache.beam.runners.core.construction.SerializablePipelineOptions; @@ -40,7 +42,9 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.flink.api.common.functions.RichMapPartitionFunction; +import org.apache.flink.api.common.functions.AbstractRichFunction; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.functions.MapPartitionFunction; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.util.Collector; @@ -53,8 +57,9 @@ * all outputs with the output number. Afterwards a filter will filter out those elements that are * not to be in a specific output. */ -public class FlinkDoFnFunction - extends RichMapPartitionFunction, WindowedValue> { +public class FlinkDoFnFunction extends AbstractRichFunction + implements FlatMapFunction, WindowedValue>, + MapPartitionFunction, WindowedValue> { private final SerializablePipelineOptions serializedOptions; @@ -71,9 +76,14 @@ public class FlinkDoFnFunction private final DoFnSchemaInformation doFnSchemaInformation; private final Map> sideInputMapping; + private transient CollectorAware collectorAware; private transient DoFnInvoker doFnInvoker; + private transient DoFnRunner doFnRunner; private transient FlinkMetricContainer metricContainer; + private boolean bundleStarted = false; + private boolean exceptionThrownInFlatMap = false; + public FlinkDoFnFunction( DoFn doFn, String stepName, @@ -100,22 +110,49 @@ public FlinkDoFnFunction( this.sideInputMapping = sideInputMapping; } + @Override + public void flatMap(WindowedValue value, Collector> out) { + try { + if (!bundleStarted) { + bundleStarted = true; + doFnRunner.startBundle(); + } + collectorAware.setCollector(out); + doFnRunner.processElement(value); + } catch (Exception e) { + exceptionThrownInFlatMap = true; + throw e; + } + } + @Override public void mapPartition( - Iterable> values, Collector> out) - throws Exception { + Iterable> values, Collector> out) { + for (WindowedValue value : values) { + flatMap(value, out); + } + } - RuntimeContext runtimeContext = getRuntimeContext(); + @Override + public void open(Configuration parameters) { + // Note that the SerializablePipelineOptions already initialize FileSystems in the readObject() + // deserialization method. However, this is a hack, and we want to properly initialize the + // options where they are needed. + FileSystems.setDefaultPipelineOptions(serializedOptions.get()); + doFnInvoker = DoFnInvokers.tryInvokeSetupFor(doFn); + metricContainer = new FlinkMetricContainer(getRuntimeContext()); - DoFnRunners.OutputManager outputManager; + // setup DoFnRunner + final RuntimeContext runtimeContext = getRuntimeContext(); + final DoFnRunners.OutputManager outputManager; if (outputMap.size() == 1) { - outputManager = new FlinkDoFnFunction.DoFnOutputManager(out); + outputManager = new DoFnOutputManager(); } else { // it has some additional outputs - outputManager = new FlinkDoFnFunction.MultiDoFnOutputManager((Collector) out, outputMap); + outputManager = new MultiDoFnOutputManager(outputMap); } - List> additionalOutputTags = Lists.newArrayList(outputMap.keySet()); + final List> additionalOutputTags = Lists.newArrayList(outputMap.keySet()); DoFnRunner doFnRunner = DoFnRunners.simpleRunner( @@ -132,79 +169,100 @@ public void mapPartition( doFnSchemaInformation, sideInputMapping); - FlinkPipelineOptions pipelineOptions = serializedOptions.get().as(FlinkPipelineOptions.class); - if (!pipelineOptions.getDisableMetrics()) { + if (!serializedOptions.get().as(FlinkPipelineOptions.class).getDisableMetrics()) { doFnRunner = new DoFnRunnerWithMetricsUpdate<>(stepName, doFnRunner, metricContainer); } - doFnRunner.startBundle(); - - for (WindowedValue value : values) { - doFnRunner.processElement(value); - } - - doFnRunner.finishBundle(); - } - - @Override - public void open(Configuration parameters) { - // Note that the SerializablePipelineOptions already initialize FileSystems in the readObject() - // deserialization method. However, this is a hack, and we want to properly initialize the - // options where they are needed. - FileSystems.setDefaultPipelineOptions(serializedOptions.get()); - doFnInvoker = DoFnInvokers.tryInvokeSetupFor(doFn); - metricContainer = new FlinkMetricContainer(getRuntimeContext()); + this.collectorAware = (CollectorAware) outputManager; + this.doFnRunner = doFnRunner; } @Override public void close() throws Exception { + Exception suppressed = null; + try { + if (bundleStarted && !exceptionThrownInFlatMap) { + doFnRunner.finishBundle(); + } + } catch (Exception e) { + // Suppress exception, so we can properly teardown DoFn. + suppressed = e; + } try { metricContainer.registerMetricsForPipelineResult(); Optional.ofNullable(doFnInvoker).ifPresent(DoFnInvoker::invokeTeardown); + if (suppressed != null) { + throw suppressed; + } } finally { Workarounds.deleteStaticCaches(); } } - static class DoFnOutputManager implements DoFnRunners.OutputManager { + interface CollectorAware { + + void setCollector(Collector> collector); + } + + static class DoFnOutputManager implements DoFnRunners.OutputManager, CollectorAware { + + @Nullable private Collector> collector; - private Collector collector; + DoFnOutputManager() { + this(null); + } - DoFnOutputManager(Collector collector) { + DoFnOutputManager(@Nullable Collector> collector) { this.collector = collector; } @Override - @SuppressWarnings("unchecked") + public void setCollector(Collector> collector) { + this.collector = Objects.requireNonNull(collector); + } + + @Override public void output(TupleTag tag, WindowedValue output) { - collector.collect( - WindowedValue.of( - new RawUnionValue(0 /* single output */, output.getValue()), - output.getTimestamp(), - output.getWindows(), - output.getPane())); + Objects.requireNonNull(collector) + .collect( + WindowedValue.of( + new RawUnionValue(0 /* single output */, output.getValue()), + output.getTimestamp(), + output.getWindows(), + output.getPane())); } } - static class MultiDoFnOutputManager implements DoFnRunners.OutputManager { + static class MultiDoFnOutputManager implements DoFnRunners.OutputManager, CollectorAware { - private Collector> collector; - private Map, Integer> outputMap; + @Nullable private Collector> collector; + private final Map, Integer> outputMap; + + MultiDoFnOutputManager(Map, Integer> outputMap) { + this.outputMap = outputMap; + } MultiDoFnOutputManager( - Collector> collector, Map, Integer> outputMap) { + @Nullable Collector> collector, + Map, Integer> outputMap) { this.collector = collector; this.outputMap = outputMap; } + @Override + public void setCollector(Collector> collector) { + this.collector = Objects.requireNonNull(collector); + } + @Override public void output(TupleTag tag, WindowedValue output) { - collector.collect( - WindowedValue.of( - new RawUnionValue(outputMap.get(tag), output.getValue()), - output.getTimestamp(), - output.getWindows(), - output.getPane())); + Objects.requireNonNull(collector) + .collect( + WindowedValue.of( + new RawUnionValue(outputMap.get(tag), output.getValue()), + output.getTimestamp(), + output.getWindows(), + output.getPane())); } } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExplodeWindowsFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExplodeWindowsFunction.java new file mode 100644 index 000000000000..b307878fb025 --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExplodeWindowsFunction.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.functions; + +import org.apache.beam.sdk.util.WindowedValue; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.util.Collector; + +/** + * Explode {@link WindowedValue} that belongs to multiple windows into multiple "single window" + * {@link WindowedValue values}, so we can safely group elements by (K, W) tuples. + * + * @param Value type. + */ +public class FlinkExplodeWindowsFunction + implements FlatMapFunction, WindowedValue> { + + @Override + public void flatMap(WindowedValue value, Collector> coll) { + value.explodeWindows().forEach(coll::collect); + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java new file mode 100644 index 000000000000..21a9cac71283 --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.functions; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.transforms.windowing.PaneInfo; +import org.apache.beam.sdk.util.WindowedValue; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.WindowingStrategy; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; +import org.apache.flink.api.common.functions.GroupReduceFunction; +import org.apache.flink.util.Collector; +import org.joda.time.Instant; + +/** + * Reduce function for non-merging GBK implementation. Implementation tries to return non-iterable + * results when possible, so we do not have to materialize all values for a single key in memory. + * + * @param Key type. + * @param Input type. + */ +public class FlinkNonMergingReduceFunction + implements GroupReduceFunction< + WindowedValue>, WindowedValue>>> { + + private static class OnceIterable implements Iterable { + + private final Iterator iterator; + + private final AtomicBoolean used = new AtomicBoolean(false); + + OnceIterable(Iterator iterator) { + this.iterator = iterator; + } + + @Override + public Iterator iterator() { + if (used.compareAndSet(false, true)) { + return iterator; + } + throw new IllegalStateException( + "GBK result is not re-iterable. You can enable re-iterations by setting '--reIterableGroupByKeyResult'."); + } + } + + private final WindowingStrategy windowingStrategy; + private final boolean reIterableResult; + + public FlinkNonMergingReduceFunction( + WindowingStrategy windowingStrategy, boolean reIterableResult) { + this.windowingStrategy = windowingStrategy; + this.reIterableResult = reIterableResult; + } + + @Override + public void reduce( + Iterable>> input, + Collector>>> coll) { + final PeekingIterator>> iterator = + Iterators.peekingIterator(input.iterator()); + final WindowedValue> first = iterator.peek(); + final BoundedWindow window = Iterables.getOnlyElement(first.getWindows()); + @SuppressWarnings("unchecked") + final Instant outputTimestamp = + ((WindowingStrategy) windowingStrategy) + .getWindowFn() + .getOutputTime(first.getTimestamp(), window); + final Instant combinedTimestamp = + windowingStrategy.getTimestampCombiner().assign(window, outputTimestamp); + final Iterable values; + if (reIterableResult) { + final List lst = new ArrayList<>(); + iterator.forEachRemaining(wv -> lst.add(wv.getValue().getValue())); + values = lst; + } else { + values = + new OnceIterable<>( + Iterators.transform( + iterator, + (WindowedValue> wv) -> + Objects.requireNonNull(wv).getValue().getValue())); + } + coll.collect( + WindowedValue.of( + KV.of(first.getValue().getKey(), values), + combinedTimestamp, + first.getWindows(), + PaneInfo.ON_TIME_AND_ONLY_FIRING)); + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java index 1dc13fe2686d..fdc0df297ddf 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java @@ -42,6 +42,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFnSchemaInformation; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.join.RawUnionValue; import org.apache.beam.sdk.transforms.reflect.DoFnInvoker; import org.apache.beam.sdk.transforms.reflect.DoFnInvokers; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; @@ -59,7 +60,7 @@ /** A {@link RichGroupReduceFunction} for stateful {@link ParDo} in Flink Batch Runner. */ public class FlinkStatefulDoFnFunction - extends RichGroupReduceFunction>, WindowedValue> { + extends RichGroupReduceFunction>, WindowedValue> { private final DoFn, OutputT> dofn; private String stepName; @@ -104,7 +105,7 @@ public FlinkStatefulDoFnFunction( @Override public void reduce( - Iterable>> values, Collector> out) + Iterable>> values, Collector> out) throws Exception { RuntimeContext runtimeContext = getRuntimeContext(); @@ -113,7 +114,7 @@ public void reduce( outputManager = new FlinkDoFnFunction.DoFnOutputManager(out); } else { // it has some additional Outputs - outputManager = new FlinkDoFnFunction.MultiDoFnOutputManager((Collector) out, outputMap); + outputManager = new FlinkDoFnFunction.MultiDoFnOutputManager(out, outputMap); } final Iterator>> iterator = values.iterator(); diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java new file mode 100644 index 000000000000..4158c026e203 --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.types; + +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.util.WindowedValue; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; + +/** + * {@link KeySelector} that extracts the key from a {@link KV} and returns it in encoded form as a + * {@code byte} array. + */ +public class WindowedKvKeySelector + implements KeySelector>, byte[]>, ResultTypeQueryable { + + private final Coder keyCoder; + private final Coder windowCoder; + + public WindowedKvKeySelector(Coder keyCoder, Coder windowCoder) { + this.keyCoder = keyCoder; + this.windowCoder = windowCoder; + } + + @Override + public byte[] getKey(WindowedValue> value) throws Exception { + final byte[] encodedKey = CoderUtils.encodeToByteArray(keyCoder, value.getValue().getKey()); + @SuppressWarnings("unchecked") + final byte[] encodedWindow = + CoderUtils.encodeToByteArray( + (Coder) windowCoder, Iterables.getOnlyElement(value.getWindows())); + return Bytes.concat(encodedKey, encodedWindow); + } + + @Override + public TypeInformation getProducedType() { + return new EncodedValueTypeInformation(); + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/CountingPipelineVisitor.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/CountingPipelineVisitor.java new file mode 100644 index 000000000000..5f92d9fd652c --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/CountingPipelineVisitor.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.utils; + +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.runners.TransformHierarchy; +import org.apache.beam.sdk.values.PValue; + +/** Pipeline visitors that fills a lookup table of {@link PValue} to number of consumers. */ +public class CountingPipelineVisitor extends Pipeline.PipelineVisitor.Defaults { + + private final Map numConsumers = new HashMap<>(); + + @Override + public void visitPrimitiveTransform(TransformHierarchy.Node node) { + for (PValue input : node.getInputs().values()) { + numConsumers.merge(input, 1, Integer::sum); + } + } + + /** + * Calculate number of consumers of a given {@link PValue}. + * + * @param value PValue to perform calculation for. + * @return Number of consumers. + */ + public int getNumConsumers(PValue value) { + return numConsumers.get(value); + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java new file mode 100644 index 000000000000..662e4d666a6b --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.beam.runners.core.construction.TransformInputs; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.runners.AppliedPTransform; +import org.apache.beam.sdk.runners.TransformHierarchy; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; + +/** + * Pipeline visitor that fills lookup table of {@link PTransform} to {@link AppliedPTransform} for + * usage in {@link + * org.apache.beam.runners.flink.FlinkBatchPortablePipelineTranslator.BatchTranslationContext}. + */ +public class LookupPipelineVisitor extends Pipeline.PipelineVisitor.Defaults { + + private final Map, AppliedPTransform> lookupTable = new HashMap<>(); + + @Override + public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) { + if ((node.getTransform()) != null) { + final AppliedPTransform applied = node.toAppliedPTransform(getPipeline()); + lookupTable.put(applied.getTransform(), applied); + } + return CompositeBehavior.ENTER_TRANSFORM; + } + + @Override + public void visitPrimitiveTransform(TransformHierarchy.Node node) { + final AppliedPTransform applied = node.toAppliedPTransform(getPipeline()); + lookupTable.put(applied.getTransform(), applied); + } + + private + AppliedPTransform> applied( + PTransform transform) { + @SuppressWarnings("unchecked") + final AppliedPTransform> applied = + (AppliedPTransform>) + lookupTable.get(transform); + if (applied == null) { + throw new IllegalArgumentException( + String.format("AppliedPTransform for %s does not exist.", transform)); + } + return applied; + } + + public Map, PValue> getInputs(PTransform transform) { + return applied(transform).getInputs(); + } + + @SuppressWarnings("unchecked") + public T getInput(PTransform transform) { + return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(applied(transform))); + } + + public Map, PValue> getOutputs(PTransform transform) { + return applied(transform).getOutputs(); + } + + @SuppressWarnings("unchecked") + public T getOutput(PTransform transform) { + return (T) Iterables.getOnlyElement(applied(transform).getOutputs().values()); + } + + @SuppressWarnings("unchecked") + public Map, Coder> getOutputCoders(PTransform transform) { + return getOutputs(transform).entrySet().stream() + .filter(e -> e.getValue() instanceof PCollection) + .collect(Collectors.toMap(Map.Entry::getKey, e -> ((PCollection) e.getValue()).getCoder())); + } +} diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java new file mode 100644 index 000000000000..935ca21e6bf0 --- /dev/null +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.batch; + +import java.util.Arrays; +import java.util.Objects; +import org.apache.beam.runners.flink.FlinkCapabilities; +import org.apache.beam.runners.flink.FlinkPipelineOptions; +import org.apache.beam.runners.flink.FlinkTestPipeline; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.flink.test.util.AbstractTestBase; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +public class NonMergingGroupByKeyTest extends AbstractTestBase { + + private static class ReiterateDoFn extends DoFn>, Void> { + + @ProcessElement + public void processElement(@Element KV> el) { + el.getValue().iterator(); + // this should throw an exception + el.getValue().iterator(); + } + } + + @Test + public void testDisabledReIterationThrowsAnException() { + // If output during closing is not supported, we can not chain DoFns and results + // are therefore materialized during output serialization. + Assume.assumeTrue(FlinkCapabilities.supportsOutputDuringClosing()); + final Pipeline p = FlinkTestPipeline.createForBatch(); + p.apply(Create.of(Arrays.asList(KV.of("a", 1), KV.of("b", 2), KV.of("c", 3)))) + .apply(GroupByKey.create()) + .apply(ParDo.of(new ReiterateDoFn<>())); + Pipeline.PipelineExecutionException resultException = null; + try { + p.run().waitUntilFinish(); + } catch (Pipeline.PipelineExecutionException exception) { + resultException = exception; + } + Assert.assertEquals( + IllegalStateException.class, Objects.requireNonNull(resultException).getCause().getClass()); + Assert.assertTrue( + resultException.getCause().getMessage().contains("GBK result is not re-iterable.")); + } + + @Test + public void testEnabledReIterationDoesNotThrowAnException() { + final Pipeline p = FlinkTestPipeline.createForBatch(); + p.getOptions().as(FlinkPipelineOptions.class).setReIterableGroupByKeyResult(true); + p.apply(Create.of(Arrays.asList(KV.of("a", 1), KV.of("b", 2), KV.of("c", 3)))) + .apply(GroupByKey.create()) + .apply(ParDo.of(new ReiterateDoFn<>())); + final PipelineResult.State state = p.run().waitUntilFinish(); + Assert.assertEquals(PipelineResult.State.DONE, state); + } +} diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index a4ea1c952094..a6358c120435 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -55,13 +55,13 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; -import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.runners.core.construction.BeamUrns; import org.apache.beam.runners.core.construction.CoderTranslation; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.runners.core.construction.DeduplicatedFlattenFactory; import org.apache.beam.runners.core.construction.EmptyFlattenAsCreateFactory; import org.apache.beam.runners.core.construction.Environments; @@ -210,8 +210,8 @@ public class DataflowRunner extends PipelineRunner { @VisibleForTesting static final int GCS_UPLOAD_BUFFER_SIZE_BYTES_DEFAULT = 1024 * 1024; - @VisibleForTesting static final String PIPELINE_FILE_FORMAT = "pipeline-%s.pb"; - @VisibleForTesting static final String DATAFLOW_GRAPH_FILE_FORMAT = "dataflow_graph-%s.json"; + @VisibleForTesting static final String PIPELINE_FILE_NAME = "pipeline.pb"; + @VisibleForTesting static final String DATAFLOW_GRAPH_FILE_NAME = "dataflow_graph.json"; private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -892,10 +892,7 @@ public DataflowPipelineJob run(Pipeline pipeline) { LOG.info("Staging pipeline description to {}", options.getStagingLocation()); byte[] serializedProtoPipeline = jobSpecification.getPipelineProto().toByteArray(); DataflowPackage stagedPipeline = - options - .getStager() - .stageToFile( - serializedProtoPipeline, String.format(PIPELINE_FILE_FORMAT, UUID.randomUUID())); + options.getStager().stageToFile(serializedProtoPipeline, PIPELINE_FILE_NAME); dataflowOptions.setPipelineUrl(stagedPipeline.getLocation()); if (!isNullOrEmpty(dataflowOptions.getDataflowWorkerJar())) { @@ -995,7 +992,7 @@ public DataflowPipelineJob run(Pipeline pipeline) { .getStager() .stageToFile( DataflowPipelineTranslator.jobToString(newJob).getBytes(UTF_8), - String.format(DATAFLOW_GRAPH_FILE_FORMAT, UUID.randomUUID())); + DATAFLOW_GRAPH_FILE_NAME); newJob.getSteps().clear(); newJob.setStepsLocation(stagedGraph.getLocation()); } @@ -1531,7 +1528,8 @@ private Coder getCoder() throws IOException { (Coder) CoderTranslation.fromProto( coderSpec.getCoder(), - RehydratedComponents.forComponents(coderSpec.getComponents())); + RehydratedComponents.forComponents(coderSpec.getComponents()), + TranslationContext.DEFAULT); } return coder; } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java index e81ab43fbb40..9a63e00ba13a 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java @@ -192,6 +192,18 @@ public Dataflow create(PipelineOptions options) { void setDumpHeapOnOOM(boolean dumpHeapBeforeExit); + /** + * The size of the worker's in-memory cache, in megabytes. + * + *

Currently, this cache is used for storing read values of side inputs. as well as the state + * for streaming jobs. + */ + @Description("The size of the worker's in-memory cache, in megabytes.") + @Default.Integer(100) + Integer getWorkerCacheMb(); + + void setWorkerCacheMb(Integer value); + /** * CAUTION: This option implies dumpHeapOnOOM, and has similar caveats. Specifically, heap dumps * can of comparable size to the default boot disk. Consider increasing the boot disk size before diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerHarnessOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerHarnessOptions.java index 3f37c74929fb..596b26dc05b8 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerHarnessOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerHarnessOptions.java @@ -17,7 +17,6 @@ */ package org.apache.beam.runners.dataflow.options; -import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.Hidden; @@ -41,15 +40,4 @@ public interface DataflowWorkerHarnessOptions extends DataflowPipelineOptions { String getJobId(); void setJobId(String value); - - /** - * The size of the worker's in-memory cache, in megabytes. - * - *

Currently, this cache is used for storing read values of side inputs. - */ - @Description("The size of the worker's in-memory cache, in megabytes.") - @Default.Integer(100) - Integer getWorkerCacheMb(); - - void setWorkerCacheMb(Integer value); } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java index 0b5e47828dc7..036d2dbf059f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.UUID; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -52,7 +51,7 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.util.MoreFutures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hasher; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteSource; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; @@ -360,21 +359,15 @@ List stageClasspathElements( @AutoValue public abstract static class StagedFile { - public static PackageUtil.StagedFile of( - String source, String sha256, @Nullable String destination) { + public static PackageUtil.StagedFile of(String source, String sha256, String destination) { return new AutoValue_PackageUtil_StagedFile(source, sha256, destination); } - public static PackageUtil.StagedFile of(String source, String sha256) { - return new AutoValue_PackageUtil_StagedFile(source, sha256, null); - } - /** The file to stage. */ public abstract String getSource(); /** The SHA-256 hash of the source file. */ public abstract String getSha256(); /** Staged target for this file. */ - @Nullable public abstract String getDestination(); } @@ -404,26 +397,40 @@ public static PackageAttributes forFileToStage( String.format("Non-existent file to stage: %s", file.getAbsolutePath())); } checkState(!file.isDirectory(), "Source file must not be a directory."); + String target; + // Dataflow worker jar and windmill binary can be overridden by providing files with + // predefined file names. Normally, we can use the artifact file name as same as + // the last component of GCS object resource path. However, we need special handling + // for those predefined names since they also need to be unique even in the same + // staging directory. + switch (dest) { + case "dataflow-worker.jar": + case "windmill_main": + target = + Environments.createStagingFileName( + file, Files.asByteSource(file).hash(Hashing.sha256())); + LOG.info("Staging custom {} as {}", dest, target); + break; + default: + target = dest; + } DataflowPackage destination = new DataflowPackage(); - String target = dest == null ? Environments.createStagingFileName(file) : dest; String resourcePath = FileSystems.matchNewResource(stagingPath, true) .resolve(target, StandardResolveOptions.RESOLVE_FILE) .toString(); destination.setLocation(resourcePath); - destination.setName(target); + destination.setName(dest); return new AutoValue_PackageUtil_PackageAttributes( file, null, destination, file.length(), hash); } public static PackageAttributes forBytesToStage( byte[] bytes, String targetName, String stagingPath) { - - Hasher hasher = Hashing.sha256().newHasher(); - String hash = hasher.putBytes(bytes).hash().toString(); + HashCode hashCode = Hashing.sha256().newHasher().putBytes(bytes).hash(); long size = bytes.length; - String target = targetName == null ? UUID.randomUUID().toString() : targetName; + String target = Environments.createStagingFileName(new File(targetName), hashCode); String resourcePath = FileSystems.matchNewResource(stagingPath, true) @@ -433,7 +440,8 @@ public static PackageAttributes forBytesToStage( targetPackage.setName(target); targetPackage.setLocation(resourcePath); - return new AutoValue_PackageUtil_PackageAttributes(null, bytes, targetPackage, size, hash); + return new AutoValue_PackageUtil_PackageAttributes( + null, bytes, targetPackage, size, hashCode.toString()); } public PackageAttributes withPackageName(String overridePackageName) { diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java index fb07652f38a1..468ec958334e 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.stream.Collectors; +import org.apache.beam.runners.core.construction.Environments; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; @@ -39,9 +41,12 @@ public static void main(String[] args) { .map( (String source) -> { try { + File file = new File(source); + HashCode hashCode = Files.asByteSource(file).hash(Hashing.sha256()); return PackageUtil.StagedFile.of( source, - Files.asByteSource(new File(source)).hash(Hashing.sha256()).toString()); + hashCode.toString(), + Environments.createStagingFileName(file, hashCode)); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java index 2847279babff..535fbce76355 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java @@ -66,6 +66,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import javax.annotation.Nullable; +import org.apache.beam.runners.core.construction.Environments; import org.apache.beam.runners.dataflow.util.PackageUtil.PackageAttributes; import org.apache.beam.runners.dataflow.util.PackageUtil.StagedFile; import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; @@ -85,6 +86,7 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.LineReader; @@ -140,12 +142,12 @@ private File makeFileWithContents(String name, String contents) throws Exception private static PackageAttributes makePackageAttributes( File file, @Nullable String overridePackageName) throws IOException { - File sourceFile = file.isDirectory() ? zipDirectory(file) : file; + StagedFile stagedFile = makeStagedFile(file.getPath()); PackageAttributes attributes = PackageUtil.PackageAttributes.forFileToStage( - sourceFile.getPath(), - Files.asByteSource(sourceFile).hash(Hashing.sha256()).toString(), - null, + stagedFile.getSource(), + stagedFile.getSha256(), + stagedFile.getDestination(), STAGING_PATH); if (overridePackageName != null) { attributes = attributes.withPackageName(overridePackageName); @@ -160,15 +162,17 @@ private static StagedFile makeStagedFile(String source) throws IOException { private static StagedFile makeStagedFile(String source, String destName) throws IOException { File file = new File(source); File sourceFile; - String sha256; + HashCode hashCode; if (file.exists()) { sourceFile = file.isDirectory() ? zipDirectory(file) : file; - sha256 = Files.asByteSource(sourceFile).hash(Hashing.sha256()).toString(); + hashCode = Files.asByteSource(sourceFile).hash(Hashing.sha256()); } else { sourceFile = file; - sha256 = ""; + hashCode = Hashing.sha256().hashBytes(new byte[] {}); } - return StagedFile.of(sourceFile.getPath(), sha256, destName); + String destination = + destName == null ? Environments.createStagingFileName(file, hashCode) : destName; + return StagedFile.of(sourceFile.getPath(), hashCode.toString(), destination); } private static File zipDirectory(File directory) throws IOException { @@ -218,8 +222,8 @@ public void testPackageNamingWithFilesHavingSameContentsAndSameNames() throws Ex makeFileWithContents("folder2/folderA/sameName", "This is a test!"); DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDestination(); - assertNotEquals(target1.getName(), target2.getName()); - assertNotEquals(target1.getLocation(), target2.getLocation()); + assertEquals(target1.getName(), target2.getName()); + assertEquals(target1.getLocation(), target2.getLocation()); } @Test diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/FnApiWindowMappingFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/FnApiWindowMappingFn.java index 07f032416167..d8889b936741 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/FnApiWindowMappingFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/FnApiWindowMappingFn.java @@ -35,6 +35,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; import org.apache.beam.model.pipeline.v1.RunnerApi.PCollection; import org.apache.beam.runners.core.construction.CoderTranslation; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.runners.core.construction.RehydratedComponents; import org.apache.beam.runners.core.construction.SdkComponents; import org.apache.beam.runners.fnexecution.control.InstructionRequestHandler; @@ -136,11 +137,15 @@ public static CacheKey create(FunctionSpec windowMappingFn, BoundedWindow mainWi outboundCoder = (Coder) CoderTranslation.fromProto( - components.getCodersOrThrow(mainInputWindowCoderId), rehydratedComponents); + components.getCodersOrThrow(mainInputWindowCoderId), + rehydratedComponents, + TranslationContext.DEFAULT); inboundCoder = (Coder) CoderTranslation.fromProto( - components.getCodersOrThrow(sideInputWindowCoderId), rehydratedComponents); + components.getCodersOrThrow(sideInputWindowCoderId), + rehydratedComponents, + TranslationContext.DEFAULT); } catch (IOException e) { throw new IllegalStateException( "Unable to create side input window mapping process bundle specification.", e); diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java index ffb617053ce1..86df406613de 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java @@ -39,6 +39,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.SdkHarnessOptions; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.state.BagState; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; @@ -97,6 +98,9 @@ static ParDoFn create( Receiver receiver, @Nullable StepContext stepContext) throws Exception { + long maxSizeBytes = + options.as(SdkHarnessOptions.class).getGroupingTableMaxSizeMb() * (1024L * 1024L); + Coder keyCoder = inputElementCoder.getKeyCoder(); Coder valueCoder = inputElementCoder.getValueCoder(); if (combineFn == null) { @@ -108,7 +112,8 @@ static ParDoFn create( PairInfo.create(), new CoderSizeEstimator<>(WindowedValue.getValueOnlyCoder(keyCoder)), new CoderSizeEstimator<>(inputCoder), - 0.001 /*sizeEstimatorSampleRate*/); + 0.001, /*sizeEstimatorSampleRate*/ + maxSizeBytes /*maxSizeBytes*/); return new SimplePartialGroupByKeyParDoFn<>(groupingTable, receiver); } else { GroupingTables.Combiner, InputT, AccumT, ?> valueCombiner = @@ -122,7 +127,8 @@ static ParDoFn create( valueCombiner, new CoderSizeEstimator<>(WindowedValue.getValueOnlyCoder(keyCoder)), new CoderSizeEstimator<>(combineFn.getAccumulatorCoder()), - 0.001 /*sizeEstimatorSampleRate*/); + 0.001, /*sizeEstimatorSampleRate*/ + maxSizeBytes /*maxSizeBytes*/); if (sideInputReader.isEmpty()) { return new SimplePartialGroupByKeyParDoFn<>(groupingTable, receiver); } else if (options.as(StreamingOptions.class).isStreaming()) { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java index 58aa30486d45..2f0871964685 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java @@ -2310,14 +2310,14 @@ public void printActiveWork(PrintWriter writer) { builder.append(""); builder.append(String.format("%016x", workItem.getShardingKey())); builder.append(""); - builder.append(workItem.getWorkToken()); + builder.append(String.format("%016x", workItem.getWorkToken())); builder.append(""); builder.append(queue.size() - 1); builder.append(""); builder.append(elapsedString(work.getStartTime(), now)); builder.append(""); builder.append(state); - builder.append("\n"); + builder.append(""); builder.append(elapsedString(work.getStateStartTime(), now)); builder.append("\n"); } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java index a3619ab9bfab..964245b43715 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java @@ -345,6 +345,8 @@ private static class WindmillValue extends SimpleWindmillState implements Val private boolean modified = false; /** Whether the in memory value is the true value. */ private boolean valueIsKnown = false; + /** The size of the encoded value */ + private long cachedSize = -1; private T value; @@ -382,6 +384,9 @@ public WindmillValue readLater() { @Override public T read() { try (Closeable scope = scopedReadState()) { + if (!valueIsKnown) { + cachedSize = -1; + } value = getFuture().get(); valueIsKnown = true; return value; @@ -397,6 +402,7 @@ public T read() { public void write(T value) { modified = true; valueIsKnown = true; + cachedSize = -1; this.value = value; } @@ -410,14 +416,18 @@ protected WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKey cache) return WorkItemCommitRequest.newBuilder().buildPartial(); } - ByteString.Output stream = ByteString.newOutput(); - if (value != null) { - coder.encode(value, stream, Coder.Context.OUTER); + ByteString encoded = null; + if (cachedSize == -1 || modified) { + ByteString.Output stream = ByteString.newOutput(); + if (value != null) { + coder.encode(value, stream, Coder.Context.OUTER); + } + encoded = stream.toByteString(); + cachedSize = encoded.size(); } - ByteString encoded = stream.toByteString(); // Place in cache to avoid a future read. - cache.put(namespace, address, this, encoded.size()); + cache.put(namespace, address, this, cachedSize); if (!modified) { // The value was read, but never written or cleared. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java index d1ab37856546..ddccbf75b188 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java @@ -59,9 +59,10 @@ public static GroupingTable> bufferingAndSampling( PairInfo pairInfo, SizeEstimator keySizer, SizeEstimator valueSizer, - double sizeEstimatorSampleRate) { + double sizeEstimatorSampleRate, + long maxSizeBytes) { return new BufferingGroupingTable<>( - DEFAULT_MAX_GROUPING_TABLE_BYTES, + maxSizeBytes, groupingKeyCreator, pairInfo, new SamplingSizeEstimator<>(keySizer, sizeEstimatorSampleRate, 1.0), @@ -94,9 +95,10 @@ public static GroupingTable combiningAndS Combiner combineFn, SizeEstimator keySizer, SizeEstimator accumulatorSizer, - double sizeEstimatorSampleRate) { + double sizeEstimatorSampleRate, + long maxSizeBytes) { return new CombiningGroupingTable<>( - DEFAULT_MAX_GROUPING_TABLE_BYTES, + maxSizeBytes, groupingKeyCreator, pairInfo, combineFn, diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle index cc0628cbe595..26a141d9c0df 100644 --- a/runners/spark/job-server/build.gradle +++ b/runners/spark/job-server/build.gradle @@ -117,6 +117,10 @@ def portableValidatesRunnerTask(String name) { excludeCategories 'org.apache.beam.sdk.testing.UsesStrictTimerOrdering' excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' }, + testFilter: { + // TODO(BEAM-10094) + excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' + }, ) } diff --git a/sdks/go/examples/stringsplit/stringsplit.go b/sdks/go/examples/stringsplit/stringsplit.go index eff55bf90cc6..d96a3417ac08 100644 --- a/sdks/go/examples/stringsplit/stringsplit.go +++ b/sdks/go/examples/stringsplit/stringsplit.go @@ -71,19 +71,7 @@ func (fn *StringSplitFn) CreateInitialRestriction(s string) offsetrange.Restrict // SplitRestriction performs initial splits so that each restriction is split // into 5. func (fn *StringSplitFn) SplitRestriction(s string, rest offsetrange.Restriction) []offsetrange.Restriction { - size := rest.End - rest.Start - splitPts := []int64{ - rest.Start, - rest.Start + (size / 5), - rest.Start + (size * 2 / 5), - rest.Start + (size * 3 / 5), - rest.Start + (size * 4 / 5), - rest.End, - } - var splits []offsetrange.Restriction - for i := 0; i < len(splitPts)-1; i++ { - splits = append(splits, offsetrange.Restriction{Start: splitPts[i], End: splitPts[i+1]}) - } + splits := rest.EvenSplits(5) log.Debugf(context.Background(), "StringSplit SplitRestrictions: %v -> %v", rest, splits) return splits } @@ -91,7 +79,7 @@ func (fn *StringSplitFn) SplitRestriction(s string, rest offsetrange.Restriction // RestrictionSize returns the size as the difference between the restriction's // start and end. func (fn *StringSplitFn) RestrictionSize(s string, rest offsetrange.Restriction) float64 { - size := float64(rest.End - rest.Start) + size := rest.Size() log.Debugf(context.Background(), "StringSplit RestrictionSize: %v -> %v", rest, size) return size } diff --git a/sdks/go/pkg/beam/core/graph/coder/coder.go b/sdks/go/pkg/beam/core/graph/coder/coder.go index 61ebdc6f3d37..2ba07d72c882 100644 --- a/sdks/go/pkg/beam/core/graph/coder/coder.go +++ b/sdks/go/pkg/beam/core/graph/coder/coder.go @@ -45,6 +45,8 @@ type CustomCoder struct { // Dec is the decoding function: []byte -> T. It may optionally take a // reflect.Type parameter and return an error as well. Dec *funcx.Fn + + ID string // (optional) This coder's ID if translated from a pipeline proto. } // TODO(herohde) 5/16/2017: do we want/need to allow user coders that follow the @@ -76,7 +78,10 @@ func (c *CustomCoder) Equals(o *CustomCoder) bool { } func (c *CustomCoder) String() string { - return fmt.Sprintf("%v[%v]", c.Type, c.Name) + if c.ID == "" { + return fmt.Sprintf("%v[%v]", c.Type, c.Name) + } + return fmt.Sprintf("%v[%v;%v]", c.Type, c.Name, c.ID) } // Type signatures of encode/decode for verification. @@ -187,6 +192,8 @@ type Coder struct { Components []*Coder // WindowedValue, KV, CoGBK Custom *CustomCoder // Custom Window *WindowCoder // WindowedValue + + ID string // (optional) This coder's ID if translated from a pipeline proto. } // Equals returns true iff the two coders are equal. It assumes that @@ -224,10 +231,16 @@ func (c *Coder) String() string { return "$" } if c.Custom != nil { - return c.Custom.String() + if c.ID == "" { + return c.Custom.String() + } + return fmt.Sprintf("%v;%v", c.Custom, c.ID) } ret := fmt.Sprintf("%v", c.Kind) + if c.ID != "" { + ret = fmt.Sprintf("%v;%v", c.Kind, c.ID) + } if len(c.Components) > 0 { var args []string for _, elm := range c.Components { diff --git a/sdks/go/pkg/beam/core/graph/fn.go b/sdks/go/pkg/beam/core/graph/fn.go index 6b4f64bd03e8..1711d7cf7db8 100644 --- a/sdks/go/pkg/beam/core/graph/fn.go +++ b/sdks/go/pkg/beam/core/graph/fn.go @@ -297,6 +297,19 @@ func NumMainInputs(num mainInputs) func(*config) { } } +// CoGBKMainInput is an optional config to NewDoFn which specifies the number +// of components of a CoGBK input to the DoFn being created, allowing for more complete +// validation. +// +// Example usage: +// var col beam.PCollection +// graph.NewDoFn(fn, graph.CoGBKMainInput(len(col.Type().Components()))) +func CoGBKMainInput(components int) func(*config) { + return func(cfg *config) { + cfg.numMainIn = mainInputs(components) + } +} + // NewDoFn constructs a DoFn from the given value, if possible. func NewDoFn(fn interface{}, options ...func(*config)) (*DoFn, error) { ret, err := NewFn(fn) @@ -586,14 +599,9 @@ func validateSideInputsNumUnknown(processFnInputs []funcx.FnParam, method *funcx // Handle cases where method has no inputs. if !ok { - if numProcessIn <= int(MainKv) { - return nil // We're good, possible for there to be no side inputs. - } - err := errors.Errorf("side inputs expected in method %v", methodName) - return errors.SetTopLevelMsgf(err, - "Missing side inputs in the %v method of a DoFn. "+ - "If side inputs are present in %v those side inputs must also be present in %v.", - methodName, processElementName, methodName) + // If there's no inputs, this is fine, as the ProcessElement method could be a + // CoGBK, and not have side inputs. + return nil } // Error if number of side inputs doesn't match any of the possible numbers of side inputs, diff --git a/sdks/go/pkg/beam/core/graph/fn_test.go b/sdks/go/pkg/beam/core/graph/fn_test.go index e5305123a4ed..f75410970fc7 100644 --- a/sdks/go/pkg/beam/core/graph/fn_test.go +++ b/sdks/go/pkg/beam/core/graph/fn_test.go @@ -26,22 +26,26 @@ import ( func TestNewDoFn(t *testing.T) { t.Run("valid", func(t *testing.T) { tests := []struct { - dfn interface{} - main mainInputs + dfn interface{} + opt func(*config) }{ - {dfn: func(string) int { return 0 }, main: MainSingle}, - {dfn: func(string, int) int { return 0 }, main: MainKv}, + {dfn: func(string) int { return 0 }, opt: NumMainInputs(MainSingle)}, + {dfn: func(string, int) int { return 0 }, opt: NumMainInputs(MainKv)}, {dfn: func(context.Context, typex.Window, typex.EventTime, reflect.Type, string, int, func(*int) bool, func() func(*int) bool, func(int)) (typex.EventTime, int, error) { return 0, 0, nil - }, main: MainKv}, - {dfn: &GoodDoFn{}, main: MainSingle}, - {dfn: &GoodDoFnOmittedMethods{}, main: MainSingle}, - {dfn: &GoodDoFnEmits{}, main: MainSingle}, - {dfn: &GoodDoFnSideInputs{}, main: MainSingle}, - {dfn: &GoodDoFnKv{}, main: MainKv}, - {dfn: &GoodDoFnKvSideInputs{}, main: MainKv}, - {dfn: &GoodDoFnAllExtras{}, main: MainKv}, - {dfn: &GoodDoFnUnexportedExtraMethod{}, main: MainSingle}, + }, opt: NumMainInputs(MainKv)}, + {dfn: &GoodDoFn{}, opt: NumMainInputs(MainSingle)}, + {dfn: &GoodDoFnOmittedMethods{}, opt: NumMainInputs(MainSingle)}, + {dfn: &GoodDoFnEmits{}, opt: NumMainInputs(MainSingle)}, + {dfn: &GoodDoFnSideInputs{}, opt: NumMainInputs(MainSingle)}, + {dfn: &GoodDoFnKv{}, opt: NumMainInputs(MainKv)}, + {dfn: &GoodDoFnKvSideInputs{}, opt: NumMainInputs(MainKv)}, + {dfn: &GoodDoFnAllExtras{}, opt: NumMainInputs(MainKv)}, + {dfn: &GoodDoFnUnexportedExtraMethod{}, opt: NumMainInputs(MainSingle)}, + {dfn: &GoodDoFnCoGbk1{}, opt: NumMainInputs(MainKv)}, + {dfn: &GoodDoFnCoGbk2{}, opt: CoGBKMainInput(3)}, + {dfn: &GoodDoFnCoGbk7{}, opt: CoGBKMainInput(8)}, + {dfn: &GoodDoFnCoGbk1wSide{}, opt: NumMainInputs(MainKv)}, } for _, test := range tests { @@ -50,8 +54,10 @@ func TestNewDoFn(t *testing.T) { if _, err := NewDoFn(test.dfn); err != nil { t.Fatalf("NewDoFn failed: %v", err) } - if _, err := NewDoFn(test.dfn, NumMainInputs(test.main)); err != nil { - t.Fatalf("NewDoFn(NumMainInputs(%v)) failed: %v", test.main, err) + if _, err := NewDoFn(test.dfn, test.opt); err != nil { + cfg := defaultConfig() + test.opt(cfg) + t.Fatalf("NewDoFn(%#v) failed: %v", cfg, err) } }) } @@ -72,10 +78,8 @@ func TestNewDoFn(t *testing.T) { {dfn: &BadDoFnMismatchedEmitsStartBundle{}}, {dfn: &BadDoFnNoEmitsFinishBundle{}}, // Validate side inputs. - {dfn: &BadDoFnNoSideInputsStartBundle{}}, {dfn: &BadDoFnMissingSideInputsStartBundle{}}, {dfn: &BadDoFnMismatchedSideInputsStartBundle{}}, - {dfn: &BadDoFnNoSideInputsFinishBundle{}}, // Validate setup/teardown. {dfn: &BadDoFnParamsInSetup{}}, {dfn: &BadDoFnParamsInTeardown{}}, @@ -121,6 +125,11 @@ func TestNewDoFn(t *testing.T) { }, main: MainKv}, {dfn: &BadDoFnAmbiguousMainInput{}, main: MainKv}, {dfn: &BadDoFnAmbiguousSideInput{}, main: MainSingle}, + // These are ambiguous with CoGBKs, but should fail with known MainInputs. + {dfn: &BadDoFnNoSideInputsStartBundle{}, main: MainSingle}, + {dfn: &BadDoFnNoSideInputsStartBundle{}, main: MainKv}, + {dfn: &BadDoFnNoSideInputsFinishBundle{}, main: MainSingle}, + {dfn: &BadDoFnNoSideInputsFinishBundle{}, main: MainKv}, } for _, test := range tests { t.Run(reflect.TypeOf(test.dfn).String(), func(t *testing.T) { @@ -368,6 +377,54 @@ func (fn *GoodDoFnKvSideInputs) StartBundle(string, func(*int) bool, func() func func (fn *GoodDoFnKvSideInputs) FinishBundle(string, func(*int) bool, func() func(*int) bool) { } +type GoodDoFnCoGbk1 struct{} + +func (fn *GoodDoFnCoGbk1) ProcessElement(int, func(*string) bool) int { + return 0 +} + +func (fn *GoodDoFnCoGbk1) StartBundle() { +} + +func (fn *GoodDoFnCoGbk1) FinishBundle() { +} + +type GoodDoFnCoGbk2 struct{} + +func (fn *GoodDoFnCoGbk2) ProcessElement(int, func(*int) bool, func(*string) bool) int { + return 0 +} + +func (fn *GoodDoFnCoGbk2) StartBundle() { +} + +func (fn *GoodDoFnCoGbk2) FinishBundle() { +} + +type GoodDoFnCoGbk7 struct{} + +func (fn *GoodDoFnCoGbk7) ProcessElement(k int, v1, v2, v3, v4, v5, v6, v7 func(*int) bool) int { + return 0 +} + +func (fn *GoodDoFnCoGbk7) StartBundle() { +} + +func (fn *GoodDoFnCoGbk7) FinishBundle() { +} + +type GoodDoFnCoGbk1wSide struct{} + +func (fn *GoodDoFnCoGbk1wSide) ProcessElement(int, func(*string) bool, func(*int) bool) int { + return 0 +} + +func (fn *GoodDoFnCoGbk1wSide) StartBundle(func(*int) bool) { +} + +func (fn *GoodDoFnCoGbk1wSide) FinishBundle(func(*int) bool) { +} + type GoodDoFnAllExtras struct{} func (fn *GoodDoFnAllExtras) ProcessElement(context.Context, typex.Window, typex.EventTime, reflect.Type, string, int, func(*int) bool, func() func(*int) bool, func(int)) (typex.EventTime, int, error) { diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go index ccc4a09acb50..f9e9a943cba8 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go +++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "math" + "sort" "sync" "time" @@ -266,33 +267,95 @@ func (n *DataSource) Progress() ProgressReportSnapshot { return ProgressReportSnapshot{PID: n.outputPID, ID: n.SID.PtransformID, Name: n.Name, Count: c} } -// Split takes a sorted set of potential split indices, selects and actuates -// split on an appropriate split index, and returns the selected split index -// if successful. Returns an error when unable to split. -func (n *DataSource) Split(splits []int64, frac float64) (int64, error) { - if splits == nil { - return 0, fmt.Errorf("failed to split: requested splits were empty") - } +// Split takes a sorted set of potential split indices and a fraction of the +// remainder to split at, selects and actuates a split on an appropriate split +// index, and returns the selected split index if successful or an error when +// unsuccessful. +// +// The bufSize param specifies the estimated number of elements that will be +// sent to this DataSource, and is used to be able to perform accurate splits +// even if the DataSource has not yet received all its elements. A bufSize of +// 0 or less indicates that its unknown, and so uses the current known size. +func (n *DataSource) Split(splits []int64, frac float64, bufSize int64) (int64, error) { if n == nil { return 0, fmt.Errorf("failed to split at requested splits: {%v}, DataSource not initialized", splits) } + if frac > 1.0 { + frac = 1.0 + } else if frac < 0.0 { + frac = 0.0 + } + n.mu.Lock() - c := n.index - // Find the smallest split index that we haven't yet processed, and set - // the promised split index to this value. - for _, s := range splits { - // // Never split on the first element, or the current element. - if s > 0 && s > c && s <= n.splitIdx { - n.splitIdx = s - fs := n.splitIdx - n.mu.Unlock() - return fs, nil - } + // Size to split within is the minimum of bufSize or splitIdx so we avoid + // including elements we already know won't be processed. + if bufSize <= 0 || n.splitIdx < bufSize { + bufSize = n.splitIdx + } + s, err := splitHelper(n.index, bufSize, splits, frac) + if err != nil { + n.mu.Unlock() + return 0, err } + n.splitIdx = s + fs := n.splitIdx n.mu.Unlock() - // If we can't find a suitable split index from the requested choices, - // return an error. - return 0, fmt.Errorf("failed to split at requested splits: {%v}, DataSource at index: %v", splits, c) + return fs, nil +} + +// splitHelper is a helper function that finds a split point in a range. +// currIdx and splitIdx should match the DataSource's index and splitIdx fields, +// and represent the start and end of the splittable range respectively. splits +// is an optional slice of valid split indices, and if nil then all indices are +// considered valid split points. frac must be between [0, 1], and represents +// a fraction of the remaining work that the split point aims to be as close +// as possible to. +func splitHelper(currIdx, splitIdx int64, splits []int64, frac float64) (int64, error) { + // Get split index from fraction. Find the closest index to the fraction of + // the remainder. + var start int64 = 0 + if currIdx > start { + start = currIdx + } + // This is the first valid split index, since we should never split at 0 or + // at the current element. + safeStart := start + 1 + // The remainder starts at our actual progress (i.e. start), but our final + // split index has to be >= our safeStart. + fracIdx := start + int64(math.Round(frac*float64(splitIdx-start))) + if fracIdx < safeStart { + fracIdx = safeStart + } + if splits == nil { + // All split points are valid so just split at fraction. + return fracIdx, nil + } else { + // Find the closest unprocessed split point to our fraction. + sort.Slice(splits, func(i, j int) bool { return splits[i] < splits[j] }) + var prevDiff int64 = math.MaxInt64 + var bestS int64 = -1 + for _, s := range splits { + if s >= safeStart && s <= splitIdx { + diff := intAbs(fracIdx - s) + if diff <= prevDiff { + prevDiff = diff + bestS = s + } else { + break // Stop early if the difference starts increasing. + } + } + } + if bestS != -1 { + return bestS, nil + } + } + return 0, fmt.Errorf("failed to split DataSource (at index: %v) at requested splits: {%v}", currIdx, splits) +} + +// intAbs implements absolute value for integers via Two's Complement. +func intAbs(n int64) int64 { + y := n >> 63 // y ← x ⟫ 63 + return (n ^ y) - y // (x ⨁ y) - y } type concatReStream struct { diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go index 1ce493cd1d8f..c0ff1a941ecc 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go +++ b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go @@ -341,7 +341,7 @@ func TestDataSource_Split(t *testing.T) { <-blockedCh // Validate that we do not split on the element we're blocking on index. // The first valid split is at test.splitIdx. - if splitIdx, err := source.Split([]int64{0, 1, 2, 3, 4, 5}, -1); err != nil { + if splitIdx, err := source.Split([]int64{0, 1, 2, 3, 4, 5}, -1, 0); err != nil { t.Errorf("error in Split: %v", err) } else if got, want := splitIdx, test.splitIdx; got != want { t.Errorf("error in Split: got splitIdx = %v, want %v ", got, want) @@ -371,6 +371,57 @@ func TestDataSource_Split(t *testing.T) { } }) + // Test that the bufSize param can be used to affect the split range. + t.Run("bufSize", func(t *testing.T) { + test := struct { + splitPts []int64 + frac float64 + bufSize int64 + splitIdx int64 + expected []interface{} + }{ + // splitIdx defaults to the max int64, so if bufSize is respected + // the closest splitPt is 3, otherwise it'll be 5000. + splitPts: []int64{3, 5000}, + frac: 0.5, + bufSize: 10, + splitIdx: 3, + expected: elements[:3], + } + + source, out, pr := initSourceTest("bufSize") + p, err := NewPlan("a", []Unit{out, source}) + if err != nil { + t.Fatalf("failed to construct plan: %v", err) + } + dc := DataContext{Data: &TestDataManager{R: pr}} + ctx := context.Background() + + // StartBundle resets the source, so no splits can be actuated before then, + // which means we need to actuate the plan manually, and insert the split request + // after StartBundle. + for i, root := range p.units { + if err := root.Up(ctx); err != nil { + t.Fatalf("error in root[%d].Up: %v", i, err) + } + } + p.status = Active + + runOnRoots(ctx, t, p, "StartBundle", func(root Root, ctx context.Context) error { return root.StartBundle(ctx, "1", dc) }) + + // SDK never splits on 0, so check that every test. + sp := SplitPoints{Splits: test.splitPts, Frac: test.frac, BufSize: test.bufSize} + if splitIdx, err := p.Split(sp); err != nil { + t.Fatalf("error in Split: %v", err) + } else if got, want := splitIdx, test.splitIdx; got != want { + t.Fatalf("error in Split: got splitIdx = %v, want %v ", got, want) + } + runOnRoots(ctx, t, p, "Process", Root.Process) + runOnRoots(ctx, t, p, "FinishBundle", Root.FinishBundle) + + validateSource(t, out, source, makeValues(test.expected...)) + }) + // Test expects splitting errors, but for processing to be successful. t.Run("errors", func(t *testing.T) { source, out, pr := initSourceTest("noSplitsUntilStarted") @@ -410,15 +461,93 @@ func TestDataSource_Split(t *testing.T) { t.Run("sanity_errors", func(t *testing.T) { var source *DataSource - if _, err := source.Split([]int64{0}, -1); err == nil { + if _, err := source.Split([]int64{0}, -1, 0); err == nil { t.Fatal("expected error splitting nil *DataSource") } - if _, err := source.Split(nil, -1); err == nil { + if _, err := source.Split(nil, -1, 0); err == nil { t.Fatal("expected error splitting nil desired splits") } }) } +// TestSplitHelper tests the underlying split logic to confirm that various +// cases produce expected split points. +func TestSplitHelper(t *testing.T) { + // Test splits at various fractions. + t.Run("SimpleSplits", func(t *testing.T) { + tests := []struct { + curr, size int64 + frac float64 + want int64 + }{ + // Split as close to the beginning as possible. + {curr: 0, size: 16, frac: 0, want: 1}, + // The closest split is at 4, even when just above or below it. + {curr: 0, size: 16, frac: 0.24, want: 4}, + {curr: 0, size: 16, frac: 0.25, want: 4}, + {curr: 0, size: 16, frac: 0.26, want: 4}, + // Split the *remainder* in half. + {curr: 0, size: 16, frac: 0.5, want: 8}, + {curr: 2, size: 16, frac: 0.5, want: 9}, + {curr: 6, size: 16, frac: 0.5, want: 11}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(%v of [%v, %v])", test.frac, test.curr, test.size), func(t *testing.T) { + got, err := splitHelper(test.curr, test.size, nil, test.frac) + if err != nil { + t.Errorf("error in splitHelper: %v", err) + } else if got != test.want { + t.Errorf("incorrect split point: got: %v, want: %v", got, test.want) + } + }) + } + }) + + // Test splits with allowed split points. + t.Run("WithAllowedSplits", func(t *testing.T) { + tests := []struct { + curr, size int64 + splits []int64 + frac float64 + want int64 + err bool // True if test should cause a failure. + }{ + // The desired split point is at 4. + {curr: 0, size: 16, splits: []int64{2, 3, 4, 5}, frac: 0.25, want: 4}, + // If we can't split at 4, choose the closest possible split point. + {curr: 0, size: 16, splits: []int64{2, 3, 5}, frac: 0.25, want: 5}, + {curr: 0, size: 16, splits: []int64{2, 3, 6}, frac: 0.25, want: 3}, + // Also test the case where all possible split points lie above or + // below the desired split point. + {curr: 0, size: 16, splits: []int64{5, 6, 7}, frac: 0.25, want: 5}, + {curr: 0, size: 16, splits: []int64{1, 2, 3}, frac: 0.25, want: 3}, + // We have progressed beyond all possible split points, so can't split. + {curr: 5, size: 16, splits: []int64{1, 2, 3}, frac: 0.25, err: true}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(%v of [%v, %v], splits = %v)", test.frac, test.curr, test.size, test.splits), func(t *testing.T) { + got, err := splitHelper(test.curr, test.size, test.splits, test.frac) + if test.err { + if err == nil { + t.Errorf("splitHelper should have errored, instead got: %v", got) + } + } else { + if err != nil { + t.Errorf("error in splitHelper: %v", err) + } else if got != test.want { + t.Errorf("incorrect split point: got: %v, want: %v", got, test.want) + } + } + }) + } + }) + + // TODO(BEAM-9935): Add SDF and element progress splitting tests from Java + // and Python once those features are added. +} + func runOnRoots(ctx context.Context, t *testing.T, p *Plan, name string, mthd func(Root, context.Context) error) { t.Helper() for i, root := range p.roots { diff --git a/sdks/go/pkg/beam/core/runtime/exec/plan.go b/sdks/go/pkg/beam/core/runtime/exec/plan.go index fde9e7c78fa0..2f07f3c3dbaf 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/plan.go +++ b/sdks/go/pkg/beam/core/runtime/exec/plan.go @@ -198,6 +198,10 @@ type SplitPoints struct { // Splits is a list of desired split indices. Splits []int64 Frac float64 + + // Estimated total number of elements (including unsent) for the source. + // A zero value indicates unknown, instead use locally known size. + BufSize int64 } // Split takes a set of potential split indexes, and if successful returns @@ -206,7 +210,7 @@ type SplitPoints struct { // Returns an error when unable to split. func (p *Plan) Split(s SplitPoints) (int64, error) { if p.source != nil { - return p.source.Split(s.Splits, s.Frac) + return p.source.Split(s.Splits, s.Frac, s.BufSize) } return 0, fmt.Errorf("failed to split at requested splits: {%v}, Source not initialized", s) } diff --git a/sdks/go/pkg/beam/core/runtime/graphx/coder.go b/sdks/go/pkg/beam/core/runtime/graphx/coder.go index b00581f0c367..212ad3b78d4c 100644 --- a/sdks/go/pkg/beam/core/runtime/graphx/coder.go +++ b/sdks/go/pkg/beam/core/runtime/graphx/coder.go @@ -106,6 +106,7 @@ func NewCoderUnmarshaller(m map[string]*pb.Coder) *CoderUnmarshaller { } } +// Coders unmarshals a list of coder ids. func (b *CoderUnmarshaller) Coders(ids []string) ([]*coder.Coder, error) { coders := make([]*coder.Coder, len(ids)) for i, id := range ids { @@ -133,6 +134,7 @@ func (b *CoderUnmarshaller) Coder(id string) (*coder.Coder, error) { if err != nil { return nil, errors.WithContextf(err, "unmarshalling coder %v", id) } + ret.ID = id b.coders[id] = ret return ret, nil @@ -256,6 +258,7 @@ func (b *CoderUnmarshaller) makeCoder(c *pb.Coder) (*coder.Coder, error) { if err != nil { return nil, err } + custom.ID = components[0] t := typex.New(custom.Type) return &coder.Coder{Kind: coder.Custom, T: t, Custom: custom}, nil diff --git a/sdks/go/pkg/beam/core/runtime/graphx/dataflow.go b/sdks/go/pkg/beam/core/runtime/graphx/dataflow.go index da6d9b520e77..364a849962cb 100644 --- a/sdks/go/pkg/beam/core/runtime/graphx/dataflow.go +++ b/sdks/go/pkg/beam/core/runtime/graphx/dataflow.go @@ -29,11 +29,12 @@ import ( // CoderRef defines the (structured) Coder in serializable form. It is // an artifact of the CloudObject encoding. type CoderRef struct { - Type string `json:"@type,omitempty"` - Components []*CoderRef `json:"component_encodings,omitempty"` - IsWrapper bool `json:"is_wrapper,omitempty"` - IsPairLike bool `json:"is_pair_like,omitempty"` - IsStreamLike bool `json:"is_stream_like,omitempty"` + Type string `json:"@type,omitempty"` + Components []*CoderRef `json:"component_encodings,omitempty"` + IsWrapper bool `json:"is_wrapper,omitempty"` + IsPairLike bool `json:"is_pair_like,omitempty"` + IsStreamLike bool `json:"is_stream_like,omitempty"` + PipelineProtoCoderID string `json:"pipeline_proto_coder_id,omitempty"` } // Exported types are used for translation lookup. @@ -92,7 +93,10 @@ func EncodeCoderRef(c *coder.Coder) (*CoderRef, error) { if err != nil { return nil, err } - return &CoderRef{Type: lengthPrefixType, Components: []*CoderRef{{Type: data}}}, nil + return &CoderRef{ + Type: lengthPrefixType, + Components: []*CoderRef{{Type: data, PipelineProtoCoderID: c.Custom.ID}}, + }, nil case coder.KV: if len(c.Components) != 2 { diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go index 81e813e4f93b..08e62f2233c3 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go +++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go @@ -38,8 +38,6 @@ type ScopedDataManager struct { mgr *DataChannelManager instID instructionID - // TODO(herohde) 7/20/2018: capture and force close open reads/writes. However, - // we would need the underlying Close to be idempotent or a separate method. closed bool mu sync.Mutex } @@ -82,9 +80,10 @@ func (s *ScopedDataManager) open(ctx context.Context, port exec.Port) (*DataChan // Close prevents new IO for this instruction. func (s *ScopedDataManager) Close() error { s.mu.Lock() + defer s.mu.Unlock() s.closed = true + s.mgr.closeInstruction(s.instID) s.mgr = nil - s.mu.Unlock() return nil } @@ -125,6 +124,14 @@ func (m *DataChannelManager) Open(ctx context.Context, port exec.Port) (*DataCha return ch, nil } +func (m *DataChannelManager) closeInstruction(instID instructionID) { + m.mu.Lock() + defer m.mu.Unlock() + for _, ch := range m.ports { + ch.removeInstruction(instID) + } +} + // clientID identifies a client of a connected channel. type clientID struct { ptransformID string @@ -148,8 +155,13 @@ type DataChannel struct { id string client dataClient - writers map[clientID]*dataWriter - readers map[clientID]*dataReader + writers map[instructionID]map[string]*dataWriter + readers map[instructionID]map[string]*dataReader + + // recently terminated instructions + endedInstructions map[instructionID]struct{} + rmQueue []instructionID + // readErr indicates a client.Recv error and is used to prevent new readers. readErr error @@ -178,11 +190,12 @@ func newDataChannel(ctx context.Context, port exec.Port) (*DataChannel, error) { func makeDataChannel(ctx context.Context, id string, client dataClient, cancelFn context.CancelFunc) *DataChannel { ret := &DataChannel{ - id: id, - client: client, - writers: make(map[clientID]*dataWriter), - readers: make(map[clientID]*dataReader), - cancelFn: cancelFn, + id: id, + client: client, + writers: make(map[instructionID]map[string]*dataWriter), + readers: make(map[instructionID]map[string]*dataReader), + endedInstructions: make(map[instructionID]struct{}), + cancelFn: cancelFn, } go ret.read(ctx) @@ -227,14 +240,16 @@ func (c *DataChannel) read(ctx context.Context) { // close the r.buf channels twice, or send on a closed channel. // Any other approach is racy, and may cause one of the above // panics. - for _, r := range c.readers { - log.Errorf(ctx, "DataChannel.read %v reader %v closing due to error on channel", c.id, r.id) - if !r.completed { - r.completed = true - r.err = err - close(r.buf) + for _, m := range c.readers { + for _, r := range m { + log.Errorf(ctx, "DataChannel.read %v reader %v closing due to error on channel", c.id, r.id) + if !r.completed { + r.completed = true + r.err = err + close(r.buf) + } + delete(cache, r.id) } - delete(cache, r.id) } c.terminateStreamOnError(err) c.mu.Unlock() @@ -266,18 +281,14 @@ func (c *DataChannel) read(ctx context.Context) { cache[id] = r } - if r.completed { - // The local reader has closed but the remote is still sending data. - // Just ignore it. We keep the reader config in the cache so we don't - // treat it as a new reader. Eventually the stream will finish and go - // through normal teardown. - continue - } // TODO(BEAM-9558): Cleanup once dataflow is updated. if len(elm.GetData()) == 0 || elm.GetIsLast() { - // Sentinel EOF segment for stream. Close buffer to signal EOF. - r.completed = true - close(r.buf) + // If this reader hasn't closed yet, do so now. + if !r.completed { + // Sentinel EOF segment for stream. Close buffer to signal EOF. + r.completed = true + close(r.buf) + } // Clean up local bookkeeping. We'll never see another message // for it again. We have to be careful not to remove the real @@ -287,6 +298,14 @@ func (c *DataChannel) read(ctx context.Context) { continue } + if r.completed { + // The local reader has closed but the remote is still sending data. + // Just ignore it. We keep the reader config in the cache so we don't + // treat it as a new reader. Eventually the stream will finish and go + // through normal teardown. + continue + } + // This send is deliberately blocking, if we exceed the buffering for // a reader. We can't buffer the entire main input, if some user code // is slow (or gets stuck). If the local side closes, the reader @@ -315,31 +334,95 @@ func (r *errReader) Close() error { // makeReader creates a dataReader. It expects to be called while c.mu is held. func (c *DataChannel) makeReader(ctx context.Context, id clientID) *dataReader { - if r, ok := c.readers[id]; ok { + var m map[string]*dataReader + var ok bool + if m, ok = c.readers[id.instID]; !ok { + m = make(map[string]*dataReader) + c.readers[id.instID] = m + } + + if r, ok := m[id.ptransformID]; ok { return r } r := &dataReader{id: id, buf: make(chan []byte, bufElements), done: make(chan bool, 1), channel: c} - c.readers[id] = r + + // Just in case initial data for an instruction arrives *after* an instructon has ended. + // eg. it was blocked by another reader being slow, or the other instruction failed. + // So we provide a pre-completed reader, and do not cache it, as there's no further cleanup for it. + if _, ok := c.endedInstructions[id.instID]; ok { + r.completed = true + close(r.buf) + r.err = io.EOF // In case of any actual data readers, so they terminate without error. + return r + } + + m[id.ptransformID] = r return r } func (c *DataChannel) removeReader(id clientID) { c.mu.Lock() - delete(c.readers, id) + if m, ok := c.readers[id.instID]; ok { + delete(m, id.ptransformID) + } c.mu.Unlock() } +const endedInstructionCap = 32 + +// removeInstruction closes all readers and writers registered for the instruction +// and deletes this instruction from the channel's reader and writer maps. +func (c *DataChannel) removeInstruction(instID instructionID) { + c.mu.Lock() + + // We don't want to leak memory, so cap the endedInstructions list. + if len(c.rmQueue) >= endedInstructionCap { + toRemove := c.rmQueue[0] + c.rmQueue = c.rmQueue[1:] + delete(c.endedInstructions, toRemove) + } + c.endedInstructions[instID] = struct{}{} + c.rmQueue = append(c.rmQueue, instID) + + rs := c.readers[instID] + ws := c.writers[instID] + + // Prevent other users while we iterate. + delete(c.readers, instID) + delete(c.writers, instID) + c.mu.Unlock() + + // Close grabs the channel lock, so this must be outside the critical section. + for _, r := range rs { + r.Close() + } + for _, w := range ws { + w.Close() + } +} + func (c *DataChannel) makeWriter(ctx context.Context, id clientID) *dataWriter { c.mu.Lock() defer c.mu.Unlock() - if w, ok := c.writers[id]; ok { + var m map[string]*dataWriter + var ok bool + if m, ok = c.writers[id.instID]; !ok { + m = make(map[string]*dataWriter) + c.writers[id.instID] = m + } + + if w, ok := m[id.ptransformID]; ok { return w } + // We don't check for ended instructions for writers, as writers + // can only be created if an instruction is in scope, and aren't + // runner or user directed. + w := &dataWriter{ch: c, id: id} - c.writers[id] = w + m[id.ptransformID] = w return w } @@ -385,9 +468,6 @@ func (r *dataReader) Read(buf []byte) (int, error) { return n, nil } -// TODO(herohde) 7/20/2018: we should probably either not be tracking writers or -// make dataWriter threadsafe. Either case is likely a corruption generator. - type dataWriter struct { buf []byte @@ -427,7 +507,7 @@ func (w *dataWriter) Close() error { // Now acquire the locks since we're sending. w.ch.mu.Lock() defer w.ch.mu.Unlock() - delete(w.ch.writers, w.id) + delete(w.ch.writers[w.id.instID], w.id.ptransformID) msg := &fnpb.Elements{ Data: []*fnpb.Elements_Data{ { diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go index 5601c3823fd4..31172e8a04b0 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go +++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "log" "strings" + "sync" "testing" "time" @@ -38,9 +39,14 @@ type fakeDataClient struct { calls int err error skipFirstError bool + + blocked sync.Mutex // Prevent data from being read by the gotourtinr. } func (f *fakeDataClient) Recv() (*fnpb.Elements, error) { + f.blocked.Lock() + defer f.blocked.Unlock() + f.calls++ data := []byte{1, 2, 3, 4, 1, 2, 3, 4} elemData := fnpb.Elements_Data{ @@ -130,6 +136,12 @@ func TestDataChannelTerminate_dataReader(t *testing.T) { // Set the 2nd Recv call to have an error. client.err = expectedError }, + }, { + name: "onInstructionEnd", + expectedError: io.EOF, + caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) { + c.removeInstruction("inst_ref") + }, }, } for _, test := range tests { @@ -170,7 +182,46 @@ func TestDataChannelTerminate_dataReader(t *testing.T) { } }) } +} + +func TestDataChannelRemoveInstruction_dataAfterClose(t *testing.T) { + done := make(chan bool, 1) + client := &fakeDataClient{t: t, done: done} + client.blocked.Lock() + + ctx, cancelFn := context.WithCancel(context.Background()) + c := makeDataChannel(ctx, "id", client, cancelFn) + c.removeInstruction("inst_ref") + + client.blocked.Unlock() + + r := c.OpenRead(ctx, "ptr", "inst_ref") + + dr := r.(*dataReader) + if !dr.completed || dr.err != io.EOF { + t.Errorf("Expected a closed reader, but was still open: completed: %v, err: %v", dr.completed, dr.err) + } + n, err := r.Read(make([]byte, 4)) + if err != io.EOF { + t.Errorf("Unexpected error from read: %v, read %d bytes.", err, n) + } +} + +func TestDataChannelRemoveInstruction_limitInstructionCap(t *testing.T) { + done := make(chan bool, 1) + client := &fakeDataClient{t: t, done: done} + ctx, cancelFn := context.WithCancel(context.Background()) + c := makeDataChannel(ctx, "id", client, cancelFn) + + for i := 0; i < endedInstructionCap+10; i++ { + instID := instructionID(fmt.Sprintf("inst_ref%d", i)) + c.OpenRead(ctx, "ptr", instID) + c.removeInstruction(instID) + } + if got, want := len(c.endedInstructions), endedInstructionCap; got != want { + t.Errorf("unexpected len(endedInstructions) got %v, want %v,", got, want) + } } func TestDataChannelTerminate_Writes(t *testing.T) { @@ -179,27 +230,34 @@ func TestDataChannelTerminate_Writes(t *testing.T) { expectedError := fmt.Errorf("EXPECTED ERROR") + instID := instructionID("inst_ref") tests := []struct { name string - caseFn func(t *testing.T, w io.WriteCloser, client *fakeDataClient) error + caseFn func(t *testing.T, w io.WriteCloser, client *fakeDataClient, c *DataChannel) error }{ { name: "onClose_Flush", - caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient) error { + caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient, c *DataChannel) error { return w.Close() }, }, { name: "onClose_Sentinel", - caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient) error { + caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient, c *DataChannel) error { client.skipFirstError = true return w.Close() }, }, { name: "onWrite", - caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient) error { + caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient, c *DataChannel) error { _, err := w.Write([]byte{'d', 'o', 'n', 'e'}) return err }, + }, { + name: "onInstructionEnd", + caseFn: func(t *testing.T, w io.WriteCloser, client *fakeDataClient, c *DataChannel) error { + c.removeInstruction(instID) + return expectedError + }, }, } for _, test := range tests { @@ -209,7 +267,7 @@ func TestDataChannelTerminate_Writes(t *testing.T) { ctx, cancelFn := context.WithCancel(context.Background()) c := makeDataChannel(ctx, "id", client, cancelFn) - w := c.OpenWrite(ctx, "ptr", "inst_ref") + w := c.OpenWrite(ctx, "ptr", instID) msg := []byte{'b', 'y', 't', 'e'} var bufSize int @@ -221,7 +279,7 @@ func TestDataChannelTerminate_Writes(t *testing.T) { } } - err := test.caseFn(t, w, client) + err := test.caseFn(t, w, client, c) if got, want := err, expectedError; err == nil || !strings.Contains(err.Error(), expectedError.Error()) { t.Errorf("Unexpected error: got %v, want %v", got, want) @@ -229,7 +287,7 @@ func TestDataChannelTerminate_Writes(t *testing.T) { // Verify that new readers return the same error for writes after stream termination. // TODO(lostluck) 2019.11.26: use the the go 1.13 errors package to check this rather // than a strings.Contains check once testing infrastructure can use go 1.13. - if n, err := c.OpenWrite(ctx, "ptr", "inst_ref").Write(msg); err != nil && !strings.Contains(err.Error(), expectedError.Error()) { + if n, err := c.OpenWrite(ctx, "ptr", instID).Write(msg); err != nil && !strings.Contains(err.Error(), expectedError.Error()) { t.Errorf("Unexpected error from write: got %v, want, %v read %d bytes.", err, expectedError, n) } select { @@ -240,5 +298,4 @@ func TestDataChannelTerminate_Writes(t *testing.T) { } }) } - } diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness.go b/sdks/go/pkg/beam/core/runtime/harness/harness.go index 26cd02e7b036..44099210d049 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/harness.go +++ b/sdks/go/pkg/beam/core/runtime/harness/harness.go @@ -310,7 +310,11 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe if ds == nil { return fail(ctx, instID, "failed to split: desired splits for root of %v was empty.", ref) } - split, err := plan.Split(exec.SplitPoints{Splits: ds.GetAllowedSplitPoints(), Frac: ds.GetFractionOfRemainder()}) + split, err := plan.Split(exec.SplitPoints{ + Splits: ds.GetAllowedSplitPoints(), + Frac: ds.GetFractionOfRemainder(), + BufSize: ds.GetEstimatedInputElements(), + }) if err != nil { return fail(ctx, instID, "unable to split %v: %v", ref, err) diff --git a/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange.go b/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange.go index 5e55f35ea592..fb1e5965431a 100644 --- a/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange.go +++ b/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange.go @@ -21,6 +21,7 @@ package offsetrange import ( "errors" + "math" "reflect" "github.com/apache/beam/sdks/go/pkg/beam" @@ -31,8 +32,43 @@ func init() { beam.RegisterType(reflect.TypeOf((*Restriction)(nil))) } +// Restriction is an offset range restriction, which represents a range of +// integers as a half-closed interval with boundaries [start, end). type Restriction struct { - Start, End int64 // Half-closed interval with boundaries [start, end). + Start, End int64 +} + +// EvenSplits splits a restriction into a number of evenly sized restrictions. +// Each split restriction is guaranteed to not be empty, and each unit from the +// original restriction is guaranteed to be contained in one split restriction. +// +// Num should be greater than 0. Otherwise there is no way to split the +// restriction and this function will return the original restriction. +func (r *Restriction) EvenSplits(num int64) (splits []Restriction) { + if num <= 1 { + // Don't split, just return original restriction. + return append(splits, *r) + } + + offset := r.Start + size := r.End - r.Start + for i := int64(0); i < num; i++ { + split := Restriction{ + Start: offset + (i * size / num), + End: offset + ((i + 1) * size / num), + } + // Skip restrictions that end up empty. + if split.End-split.Start <= 0 { + continue + } + splits = append(splits, split) + } + return splits +} + +// Size returns the restriction's size as the difference between Start and End. +func (r *Restriction) Size() float64 { + return float64(r.End - r.Start) } // Tracker tracks a restriction that can be represented as a range of integer values, @@ -42,9 +78,8 @@ type Restriction struct { type Tracker struct { Rest Restriction Claimed int64 // Tracks the last claimed position. - Stopped bool // Tracks whether TryClaim has already indicated to stop processing elements for - // any reason. - Err error + Stopped bool // Tracks whether TryClaim has indicated to stop processing elements. + Err error } // NewTracker is a constructor for an Tracker given a start and end range. @@ -57,10 +92,15 @@ func NewTracker(rest Restriction) *Tracker { } } -// TryClaim accepts an int64 position and successfully claims it if that position is greater than -// the previously claimed position and less than the end of the restriction. Note that the -// Tracker is not considered done until a position >= tracker.end tries to be claimed, -// at which point this method signals to end processing. +// TryClaim accepts an int64 position representing the starting position of a block of work. It +// successfully claims it if the position is greater than the previously claimed position and within +// the restriction. Claiming a position at or beyond the end of the restriction signals that the +// entire restriction has been processed and is now done, at which point this method signals to end +// processing. +// +// The tracker stops with an error if a claim is attempted after the tracker has signalled to stop, +// if a position is claimed before the start of the restriction, or if a position is claimed before +// the latest successfully claimed. func (tracker *Tracker) TryClaim(rawPos interface{}) bool { if tracker.Stopped == true { tracker.Err = errors.New("cannot claim work after restriction tracker returns false") @@ -88,7 +128,7 @@ func (tracker *Tracker) TryClaim(rawPos interface{}) bool { return true } -// IsDone returns true if the most recent claimed element is past the end of the restriction. +// GetError returns the error that caused the tracker to stop, if there is one. func (tracker *Tracker) GetError() error { return tracker.Err } @@ -105,7 +145,8 @@ func (tracker *Tracker) TrySplit(fraction float64) (primary, residual interface{ fraction = 1 } - splitPt := tracker.Claimed + int64(fraction*float64(tracker.Rest.End-tracker.Claimed)) + // Use Ceil to always round up from float split point. + splitPt := tracker.Claimed + int64(math.Ceil(fraction*float64(tracker.Rest.End-tracker.Claimed))) if splitPt >= tracker.Rest.End { return tracker.Rest, nil, nil } @@ -123,5 +164,5 @@ func (tracker *Tracker) GetProgress() (done, remaining float64) { // IsDone returns true if the most recent claimed element is past the end of the restriction. func (tracker *Tracker) IsDone() bool { - return tracker.Claimed >= tracker.Rest.End + return tracker.Err == nil && tracker.Claimed >= tracker.Rest.End } diff --git a/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange_test.go b/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange_test.go new file mode 100644 index 000000000000..489e25146249 --- /dev/null +++ b/sdks/go/pkg/beam/io/rtrackers/offsetrange/offsetrange_test.go @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package offsetrange + +import ( + "fmt" + "github.com/google/go-cmp/cmp" + "testing" +) + +// TestRestriction_EvenSplits tests various splits and checks that they all +// follow the contract for EvenSplits. This means that all restrictions are +// evenly split, that each restriction has at least one element, and that each +// element is present in the split restrictions. +func TestRestriction_EvenSplits(t *testing.T) { + tests := []struct { + rest Restriction + num int64 + }{ + {rest: Restriction{Start: 0, End: 21}, num: 4}, + {rest: Restriction{Start: 21, End: 42}, num: 4}, + {rest: Restriction{Start: 0, End: 5}, num: 10}, + {rest: Restriction{Start: 0, End: 21}, num: -1}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(rest[%v, %v], splits = %v)", + test.rest.Start, test.rest.End, test.num), func(t *testing.T) { + r := test.rest + + // Get the minimum size that a split restriction can be. Max size + // should be min + 1. This way we can check the size of each split. + num := test.num + if num <= 1 { + num = 1 + } + min := (r.End - r.Start) / num + + splits := r.EvenSplits(test.num) + prevEnd := r.Start + for _, split := range splits { + size := split.End - split.Start + // Check: Each restriction has at least 1 element. + if size == 0 { + t.Errorf("split restriction [%v, %v] is empty, size must be greater than 0.", + split.Start, split.End) + } + // Check: Restrictions are evenly split. + if size != min && size != min+1 { + t.Errorf("split restriction [%v, %v] has unexpected size. got: %v, want: %v or %v", + split.Start, split.End, size, min, min+1) + } + // Check: All elements are still in a split restrictions. This + // logic assumes that the splits are returned in order which + // isn't guaranteed by EvenSplits, but this check is way easier + // with the assumption. + if split.Start != prevEnd { + t.Errorf("restriction range [%v, %v] missing after splits.", + prevEnd, split.Start) + } else { + prevEnd = split.End + } + } + if prevEnd != r.End { + t.Errorf("restriction range [%v, %v] missing after splits.", + prevEnd, r.End) + } + }) + } +} + +// TestTracker_TryClaim validates both success and failure cases for TryClaim. +func TestTracker_TryClaim(t *testing.T) { + // Test that TryClaim works as expected when called correctly. + t.Run("Correctness", func(t *testing.T) { + tests := []struct { + rest Restriction + claims []int64 + }{ + {rest: Restriction{Start: 0, End: 3}, claims: []int64{0, 1, 2, 3}}, + {rest: Restriction{Start: 10, End: 40}, claims: []int64{15, 20, 50}}, + {rest: Restriction{Start: 0, End: 3}, claims: []int64{4}}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(rest[%v, %v], claims = %v)", + test.rest.Start, test.rest.End, test.claims), func(t *testing.T) { + rt := NewTracker(test.rest) + for _, pos := range test.claims { + // If TryClaim returns false, check if there was an error. + if !rt.TryClaim(pos) && !rt.IsDone() { + t.Fatalf("tracker claiming %v failed, error: %v", pos, rt.GetError()) + } + } + }) + } + }) + + // Test that each invalid error case actually results in an error. + t.Run("Errors", func(t *testing.T) { + tests := []struct { + rest Restriction + claims []int64 + }{ + // Claiming backwards. + {rest: Restriction{Start: 0, End: 3}, claims: []int64{0, 2, 1}}, + // Claiming before start of restriction. + {rest: Restriction{Start: 10, End: 40}, claims: []int64{8}}, + // Claiming after tracker signalled to stop. + {rest: Restriction{Start: 0, End: 3}, claims: []int64{4, 5}}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(rest[%v, %v], claims = %v)", + test.rest.Start, test.rest.End, test.claims), func(t *testing.T) { + rt := NewTracker(test.rest) + for _, pos := range test.claims { + // Finish successfully if we got an error. + if !rt.TryClaim(pos) && !rt.IsDone() && rt.GetError() != nil { + return + } + } + t.Fatal("tracker did not fail on invalid claim") + }) + } + }) +} + +// TestTracker_TrySplit tests that TrySplit follows its contract, meaning that +// splits don't lose any elements, split fractions are clamped to 0 or 1, and +// that TrySplit always splits at the nearest integer greater than the given +// fraction. +func TestTracker_TrySplit(t *testing.T) { + tests := []struct { + rest Restriction + claimed int64 + fraction float64 + // Index where we want the split to happen. This will be the end + // (exclusive) of the primary and first element of the residual. + splitPt int64 + }{ + { + rest: Restriction{Start: 0, End: 1}, + claimed: 0, + fraction: 0.5, + splitPt: 1, + }, + { + rest: Restriction{Start: 0, End: 5}, + claimed: 0, + fraction: 0.5, + splitPt: 3, + }, + { + rest: Restriction{Start: 0, End: 10}, + claimed: 5, + fraction: 0.5, + splitPt: 8, + }, + { + rest: Restriction{Start: 0, End: 10}, + claimed: 5, + fraction: -0.5, + splitPt: 5, + }, + { + rest: Restriction{Start: 0, End: 10}, + claimed: 5, + fraction: 1.5, + splitPt: 10, + }, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("(split at %v of [%v, %v])", + test.fraction, test.claimed, test.rest.End), func(t *testing.T) { + rt := NewTracker(test.rest) + ok := rt.TryClaim(test.claimed) + if !ok { + t.Fatalf("tracker failed on initial claim: %v", test.claimed) + } + gotP, gotR, err := rt.TrySplit(test.fraction) + if err != nil { + t.Fatalf("tracker failed on split: %v", err) + } + var wantP interface{} = Restriction{Start: test.rest.Start, End: test.splitPt} + var wantR interface{} = Restriction{Start: test.splitPt, End: test.rest.End} + if test.splitPt == test.rest.End { + wantR = nil // When residuals are empty we should get nil. + } + if !cmp.Equal(gotP, wantP) { + t.Errorf("split got incorrect primary: got: %v, want: %v", gotP, wantP) + } + if !cmp.Equal(gotR, wantR) { + t.Errorf("split got incorrect residual: got: %v, want: %v", gotR, wantR) + } + }) + } +} diff --git a/sdks/go/pkg/beam/io/synthetic/source.go b/sdks/go/pkg/beam/io/synthetic/source.go index 00033206cf62..fbe385c6bd32 100644 --- a/sdks/go/pkg/beam/io/synthetic/source.go +++ b/sdks/go/pkg/beam/io/synthetic/source.go @@ -24,10 +24,11 @@ package synthetic import ( "fmt" - "github.com/apache/beam/sdks/go/pkg/beam" - "github.com/apache/beam/sdks/go/pkg/beam/io/rtrackers/offsetrange" "math/rand" "time" + + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/io/rtrackers/offsetrange" ) // Source creates a synthetic source transform that emits randomly @@ -95,35 +96,13 @@ func (fn *sourceFn) CreateInitialRestriction(config SourceConfig) offsetrange.Re // method will contain at least one element, so the number of splits will not // exceed the number of elements. func (fn *sourceFn) SplitRestriction(config SourceConfig, rest offsetrange.Restriction) (splits []offsetrange.Restriction) { - if config.InitialSplits <= 1 { - // Don't split, just return original restriction. - return append(splits, rest) - } - - // TODO(BEAM-9978) Move this implementation of the offset range restriction - // splitting to the restriction itself, and add testing. - num := int64(config.InitialSplits) - offset := rest.Start - size := rest.End - rest.Start - for i := int64(0); i < num; i++ { - split := offsetrange.Restriction{ - Start: offset + (i * size / num), - End: offset + ((i + 1) * size / num), - } - // Skip restrictions that end up empty. - if split.End-split.Start <= 0 { - continue - } - splits = append(splits, split) - } - return splits + return rest.EvenSplits(int64(config.InitialSplits)) } // RestrictionSize outputs the size of the restriction as the number of elements // that restriction will output. func (fn *sourceFn) RestrictionSize(config SourceConfig, rest offsetrange.Restriction) float64 { - // TODO(BEAM-9978) Move this size implementation to the offset range restriction itself. - return float64(rest.End - rest.Start) + return rest.Size() } // CreateTracker just creates an offset range restriction tracker for the diff --git a/sdks/go/pkg/beam/io/synthetic/step.go b/sdks/go/pkg/beam/io/synthetic/step.go index 977d5540e410..4fc23bc15d56 100644 --- a/sdks/go/pkg/beam/io/synthetic/step.go +++ b/sdks/go/pkg/beam/io/synthetic/step.go @@ -17,10 +17,11 @@ package synthetic import ( "fmt" - "github.com/apache/beam/sdks/go/pkg/beam" - "github.com/apache/beam/sdks/go/pkg/beam/io/rtrackers/offsetrange" "math/rand" "time" + + "github.com/apache/beam/sdks/go/pkg/beam" + "github.com/apache/beam/sdks/go/pkg/beam/io/rtrackers/offsetrange" ) // Step creates a synthetic step transform that receives KV<[]byte, []byte> @@ -98,35 +99,13 @@ func (fn *sdfStepFn) CreateInitialRestriction(key, val []byte) offsetrange.Restr // method will contain at least one element, so the number of splits will not // exceed the number of elements. func (fn *sdfStepFn) SplitRestriction(key, val []byte, rest offsetrange.Restriction) (splits []offsetrange.Restriction) { - if fn.cfg.InitialSplits <= 1 { - // Don't split, just return original restriction. - return append(splits, rest) - } - - // TODO(BEAM-9978) Move this implementation of the offset range restriction - // splitting to the restriction itself, and add testing. - num := int64(fn.cfg.InitialSplits) - offset := rest.Start - size := rest.End - rest.Start - for i := int64(0); i < num; i++ { - split := offsetrange.Restriction{ - Start: offset + (i * size / num), - End: offset + ((i + 1) * size / num), - } - // Skip restrictions that end up empty. - if split.End-split.Start <= 0 { - continue - } - splits = append(splits, split) - } - return splits + return rest.EvenSplits(int64(fn.cfg.InitialSplits)) } // RestrictionSize outputs the size of the restriction as the number of elements // that restriction will output. func (fn *sdfStepFn) RestrictionSize(key, val []byte, rest offsetrange.Restriction) float64 { - // TODO(BEAM-9978) Move this size implementation to the offset range restriction itself. - return float64(rest.End - rest.Start) + return rest.Size() } // CreateTracker creates an offset range restriction tracker for the diff --git a/sdks/go/pkg/beam/pardo.go b/sdks/go/pkg/beam/pardo.go index d53e270e6a35..de3063446c79 100644 --- a/sdks/go/pkg/beam/pardo.go +++ b/sdks/go/pkg/beam/pardo.go @@ -17,6 +17,7 @@ package beam import ( "fmt" + "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder" "github.com/apache/beam/sdks/go/pkg/beam/core/typex" @@ -37,12 +38,14 @@ func TryParDo(s Scope, dofn interface{}, col PCollection, opts ...Option) ([]PCo return nil, addParDoCtx(err, s) } - num := graph.MainSingle + doFnOpt := graph.NumMainInputs(graph.MainSingle) // Check the PCollection for any keyed type (not just KV specifically). - if typex.IsKV(col.Type()) || typex.IsCoGBK(col.Type()) { - num = graph.MainKv + if typex.IsKV(col.Type()) { + doFnOpt = graph.NumMainInputs(graph.MainKv) + } else if typex.IsCoGBK(col.Type()) { + doFnOpt = graph.CoGBKMainInput(len(col.Type().Components())) } - fn, err := graph.NewDoFn(dofn, graph.NumMainInputs(num)) + fn, err := graph.NewDoFn(dofn, doFnOpt) if err != nil { return nil, addParDoCtx(err, s) } diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow.go b/sdks/go/pkg/beam/runners/dataflow/dataflow.go index 8681ceb05e25..920731338b13 100644 --- a/sdks/go/pkg/beam/runners/dataflow/dataflow.go +++ b/sdks/go/pkg/beam/runners/dataflow/dataflow.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "path" + "strings" "sync/atomic" "time" @@ -127,6 +128,17 @@ func Execute(ctx context.Context, p *beam.Pipeline) error { hooks.SerializeHooksToOptions() experiments := jobopts.GetExperiments() + // Always use runner v2, unless set already. + var v2set bool + for _, e := range experiments { + if strings.Contains(e, "use_runner_v2") || strings.Contains(e, "use_unified_worker") { + v2set = true + break + } + } + if !v2set { + experiments = append(experiments, "use_unified_worker") + } if *minCPUPlatform != "" { experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform)) } diff --git a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml index fea30fb08615..751466f5a446 100644 --- a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml +++ b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml @@ -95,6 +95,7 @@ + diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java index fcd744caf67b..a4e9c86992a5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java @@ -66,9 +66,25 @@ public Coder getElemCoder() { /** * Builds an instance of {@code IterableT}, this coder's associated {@link Iterable}-like subtype, * from a list of decoded elements. + * + *

Override {@link #decodeToIterable(List, long, InputStream)} if you need access to the + * terminator value and the {@link InputStream}. */ protected abstract IterableT decodeToIterable(List decodedElements); + /** + * Builds an instance of {@code IterableT}, this coder's associated {@link Iterable}-like subtype, + * from a list of decoded elements with the {@link InputStream} at the position where this coder + * detected the end of the stream. + */ + protected IterableT decodeToIterable( + List decodedElements, long terminatorValue, InputStream in) throws IOException { + throw new IllegalStateException( + String.format( + "%s does not support non zero terminator values. Received stream with terminator %s.", + iterableName, terminatorValue)); + } + ///////////////////////////////////////////////////////////////////////////// // Internal operations below here. @@ -136,7 +152,11 @@ public IterableT decode(InputStream inStream) throws IOException, CoderException count = VarInt.decodeLong(dataInStream); } } - return decodeToIterable(elements); + if (count == 0) { + return decodeToIterable(elements); + } else { + return decodeToIterable(elements, count, inStream); + } } @Override diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java index e02c938e68d6..19baa8d4cb0a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java @@ -725,7 +725,7 @@ public void close() throws IOException {} @Override public Instant getWatermark() { - throw new UnsupportedOperationException("getWatermark is never meant to be invoked."); + return BoundedWindow.TIMESTAMP_MAX_VALUE; } @Override @@ -836,16 +836,20 @@ public UnboundedSourceRestriction currentRestriction() { @Override public SplitResult> trySplit( double fractionOfRemainder) { - // Don't split if we have claimed all since the SDF wrapper will be finishing soon. + // Don't split if we have the empty sources since the SDF wrapper will be finishing soon. + UnboundedSourceRestriction currentRestriction = currentRestriction(); + if (currentRestriction.getSource() instanceof EmptyUnboundedSource) { + return null; + } + // Our split result sets the primary to have no checkpoint mark associated // with it since when we resume we don't have any state but we specifically pass // the checkpoint mark to the current reader so that when we finish the current bundle // we may register for finalization. - UnboundedSourceRestriction currentRestriction = currentRestriction(); SplitResult> result = SplitResult.of( UnboundedSourceRestriction.create( - EmptyUnboundedSource.INSTANCE, null, currentRestriction.getWatermark()), + EmptyUnboundedSource.INSTANCE, null, BoundedWindow.TIMESTAMP_MAX_VALUE), currentRestriction); currentReader = EmptyUnboundedSource.INSTANCE.createReader(null, currentRestriction.getCheckpoint()); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java index c5ad12a3b4c8..5f1508168d45 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java @@ -82,6 +82,20 @@ enum LogLevel { void setSdkHarnessLogLevelOverrides(SdkHarnessLogLevelOverrides value); + /** + * Size (in MB) of each grouping table used to pre-combine elements. If unset, defaults to 100 MB. + * + *

CAUTION: If set too large, workers may run into OOM conditions more easily, each worker may + * have many grouping tables in-memory concurrently. + */ + @Description( + "The size (in MB) of the grouping tables used to pre-combine elements before " + + "shuffling. Larger values may reduce the amount of data shuffled.") + @Default.Integer(100) + int getGroupingTableMaxSizeMb(); + + void setGroupingTableMaxSizeMb(int value); + /** * Defines a log level override for a specific class, package, or name. * diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java new file mode 100644 index 000000000000..16a728167b24 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.transforms; + +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.transforms.join.CoGbkResult; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.transforms.windowing.Trigger; +import org.apache.beam.sdk.transforms.windowing.WindowFn; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; + +/** + * The {@code PTransform}s that allow to compute different set functions across {@link + * PCollection}s. + * + *

They come in two variants. 1. Between two {@link PCollection} 2. Between two or more {@link + * PCollection} in a {@link PCollectionList}. + * + *

Following {@code PTransform}s follows SET DISTINCT semantics: intersectDistinct, + * expectDistinct, unionDistinct + * + *

Following {@code PTransform}s follows SET ALL semantics: intersectAll, expectAll, unionAll + * + *

For example, the following demonstrates intersectDistinct between two collections {@link + * PCollection}s. + * + *

{@code
+ * Pipeline p = ...;
+ *
+ * PCollection left = p.apply(Create.of("1", "2", "3", "3", "4", "5"));
+ * PCollection right = p.apply(Create.of("1", "3", "4", "4", "6"));
+ *
+ * PCollection results =
+ *     left.apply(SetFns.intersectDistinct(right)); // results will be PCollection containing: "1","3","4"
+ *
+ * }
+ * + *

For example, the following demonstrates intersectDistinct between three collections {@link + * PCollection}s in a {@link PCollectionList}. + * + *

{@code
+ * Pipeline p = ...;
+ *
+ * PCollection first = p.apply(Create.of("1", "2", "3", "3", "4", "5"));
+ * PCollection second = p.apply(Create.of("1", "3", "4", "4", "6"));
+ * PCollection third = p.apply(Create.of("3", "4", "4"));
+ *
+ * // Following example will perform (first intersect second) intersect third.
+ * PCollection results =
+ *     PCollectionList.of(first).and(second).and(third)
+ *     .apply(SetFns.intersectDistinct()); // results will be PCollection containing: "3","4"
+ *
+ * }
+ */ +public class Sets { + + /** + * Returns a new {@code PTransform} transform that follows SET DISTINCT semantics to compute the + * intersection with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} will all distinct elements that present in + * both pipeline is constructed and provided {@link PCollection}. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}). Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "2", "3", "3", "4", "5"));
+   * PCollection right = p.apply(Create.of("1", "3", "4", "4", "6"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.intersectDistinct(right)); // results will be PCollection containing: "1","3","4"
+   *
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> intersectDistinct( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, intersectDistinct()); + } + + /** + * Returns a {@code PTransform} that takes a {@code PCollectionList>} and returns a + * {@code PCollection} containing the intersection of collections done in order for all + * collections in {@code PCollectionList}. + * + *

Returns a new {@code PTransform} transform that follows SET DISTINCT semantics which takes a + * {@code PCollectionList>} and returns a {@code PCollection} containing the + * intersection of collections done in order for all collections in {@code PCollectionList}. + * + *

The elements of the output {@link PCollection} will have all distinct elements that are + * present in both pipeline is constructed and next {@link PCollection} in the list and applied to + * all collections in order. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection first = p.apply(Create.of("1", "2", "3", "3", "4", "5"));
+   * PCollection second = p.apply(Create.of("1", "3", "4", "4", "6"));
+   * PCollection third = p.apply(Create.of("3", "4", "4"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.intersectDistinct()); // results will be PCollection containing: "3","4"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static PTransform, PCollection> intersectDistinct() { + SerializableBiFunction intersectFn = + (numberOfElementsinLeft, numberOfElementsinRight) -> + (numberOfElementsinLeft > 0 && numberOfElementsinRight > 0) ? 1L : 0L; + return new SetImplCollections<>(intersectFn); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics to compute the + * intersection with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} which will follow INTESECT_ALL Semantics as + * follows: Given there are m elements on pipeline which is constructed {@link PCollection} (left) + * and n elements on in provided {@link PCollection} (right): - it will output MIN(m - n, 0) + * elements of left for all elements which are present in both left and right. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "1", "1", "2", "3", "3", "4", "5"));
+   * PCollection right = p.apply(Create.of("1", "1", "3", "4", "4", "6"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.intersectAll(right)); // results will be PCollection containing: "1","1","3","4"
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> intersectAll( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, intersectAll()); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics which takes a {@code + * PCollectionList>} and returns a {@code PCollection} containing the + * intersection all of collections done in order for all collections in {@code + * PCollectionList}. + * + *

The elements of the output {@link PCollection} which will follow INTERSECT_ALL semantics. + * Output is calculated as follows: Given there are m elements on pipeline which is constructed + * {@link PCollection} (left) and n elements on in provided {@link PCollection} (right): - it will + * output MIN(m - n, 0) elements of left for all elements which are present in both left and + * right. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   * PCollection first = p.apply(Create.of("1", "1", "1", "2", "3", "3", "4", "5"));
+   * PCollection second = p.apply(Create.of("1", "1", "3", "4", "4", "6"));
+   * PCollection third = p.apply(Create.of("1", "5"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.intersectAll()); // results will be PCollection containing: "1"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static PTransform, PCollection> intersectAll() { + return new SetImplCollections<>(Math::min); + } + + /** + * Returns a new {@code PTransform} transform that follows SET DISTINCT semantics to compute the + * difference (except) with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} will all distinct elements that present in + * pipeline is constructed but not present in provided {@link PCollection}. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "1", "1", "2", "3", "3","4", "5"));
+   * PCollection right = p.apply(Create.of("1", "1", "3", "4", "4", "6"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.exceptDistinct(right)); // results will be PCollection containing: "2","5"
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> exceptDistinct( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, exceptDistinct()); + } + + /** + * Returns a {@code PTransform} that takes a {@code PCollectionList>} and returns a + * {@code PCollection} containing the difference (except) of collections done in order for all + * collections in {@code PCollectionList}. + * + *

Returns a new {@code PTransform} transform that follows SET DISTINCT semantics which takes a + * {@code PCollectionList>} and returns a {@code PCollection} containing the + * difference (except) of collections done in order for all collections in {@code + * PCollectionList}. + * + *

The elements of the output {@link PCollection} will have all distinct elements that are + * present in pipeline is constructed but not present in next {@link PCollection} in the list and + * applied to all collections in order. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   * PCollection first = p.apply(Create.of("1", "1", "1", "2", "3", "3", "4", "5"));
+   * PCollection second = p.apply(Create.of("1", "1", "3", "4", "4", "6"));
+   *
+   * PCollection third = p.apply(Create.of("1", "2", "2"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.exceptDistinct()); // results will be PCollection containing: "5"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static PTransform, PCollection> exceptDistinct() { + SerializableBiFunction exceptFn = + (numberOfElementsinLeft, numberOfElementsinRight) -> + numberOfElementsinLeft > 0 && numberOfElementsinRight == 0 ? 1L : 0L; + return new SetImplCollections<>(exceptFn); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics to compute the + * difference all (exceptAll) with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} which will follow EXCEPT_ALL Semantics as + * follows: Given there are m elements on pipeline which is constructed {@link PCollection} (left) + * and n elements on in provided {@link PCollection} (right): - it will output m elements of left + * for all elements which are present in left but not in right. - it will output MAX(m - n, 0) + * elements of left for all elements which are present in both left and right. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "1", "1", "2", "3", "3", "3", "4", "5"));
+   * PCollection right = p.apply(Create.of("1", "3", "4", "4", "6"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.exceptAll(right)); // results will be PCollection containing: "1","1","2","3","3","5"
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> exceptAll( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, exceptAll()); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics which takes a {@code + * PCollectionList>} and returns a {@code PCollection} containing the difference + * all (exceptAll) of collections done in order for all collections in {@code PCollectionList}. + * + *

The elements of the output {@link PCollection} which will follow EXCEPT_ALL semantics. + * Output is calculated as follows: Given there are m elements on pipeline which is constructed + * {@link PCollection} (left) and n elements on in provided {@link PCollection} (right): - it will + * output m elements of left for all elements which are present in left but not in right. - it + * will output MAX(m - n, 0) elements of left for all elements which are present in both left and + * right. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   * PCollection first = p.apply(Create.of("1", "1", "1", "2", "3", "3", "3", "4", "5"));
+   * PCollection second = p.apply(Create.of("1", "3", "4", "4", "6"));
+   * PCollection third = p.apply(Create.of("1", "5"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.exceptAll()); // results will be PCollection containing: "1","2","3","3"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static PTransform, PCollection> exceptAll() { + SerializableBiFunction exceptFn = + (numberOfElementsinLeft, numberOfElementsinRight) -> + Math.max(numberOfElementsinLeft - numberOfElementsinRight, 0L); + return new SetImplCollections<>(exceptFn); + } + + /** + * Returns a new {@code PTransform} transform that follows SET DISTINCT semantics to compute the + * union with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} will all distinct elements that present in + * pipeline is constructed or present in provided {@link PCollection}. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "1", "2"));
+   * PCollection right = p.apply(Create.of("1", "3", "4", "4"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.unionDistinct(right)); // results will be PCollection containing: "1","2","3","4"
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> unionDistinct( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, unionDistinct()); + } + + /** + * Returns a new {@code PTransform} transform that follows SET DISTINCT semantics which takes a + * {@code PCollectionList>} and returns a {@code PCollection} containing the + * union of collections done in order for all collections in {@code PCollectionList}. + * + *

The elements of the output {@link PCollection} will have all distinct elements that are + * present in pipeline is constructed or present in next {@link PCollection} in the list and + * applied to all collections in order. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   * PCollection first = p.apply(Create.of("1", "1", "2"));
+   * PCollection second = p.apply(Create.of("1", "3", "4", "4"));
+   *
+   * PCollection third = p.apply(Create.of("1", "5"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.unionDistinct()); // results will be PCollection containing: "1","2","3","4","5"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static PTransform, PCollection> unionDistinct() { + SerializableBiFunction unionFn = + (numberOfElementsinLeft, numberOfElementsinRight) -> 1L; + return new SetImplCollections<>(unionFn); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics to compute the + * unionAll with provided {@code PCollection}. + * + *

The argument should not be modified after this is called. + * + *

The elements of the output {@link PCollection} which will follow UNION_ALL semantics as + * follows: Given there are m elements on pipeline which is constructed {@link PCollection} (left) + * and n elements on in provided {@link PCollection} (right): - it will output m elements of left + * and m elements of right. + * + *

Note that this transform requires that the {@code Coder} of the all {@code PCollection} + * to be deterministic (see {@link Coder#verifyDeterministic()}). If the collection {@code Coder} + * is not deterministic, an exception is thrown at pipeline construction time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the input {@code PCollection} + * + *

{@code
+   * Pipeline p = ...;
+   *
+   * PCollection left = p.apply(Create.of("1", "1", "2"));
+   * PCollection right = p.apply(Create.of("1", "3", "4", "4"));
+   *
+   * PCollection results =
+   *     left.apply(SetFns.unionAll(right)); // results will be PCollection containing: "1","1","1","2","3","4","4"
+   * }
+ * + * @param the type of the elements in the input and output {@code PCollection}s. + */ + public static PTransform, PCollection> unionAll( + PCollection rightCollection) { + checkNotNull(rightCollection, "rightCollection argument is null"); + return new SetImpl<>(rightCollection, unionAll()); + } + + /** + * Returns a new {@code PTransform} transform that follows SET ALL semantics which takes a {@code + * PCollectionList>} and returns a {@code PCollection} containing the unionAll + * of collections done in order for all collections in {@code PCollectionList}. + * + *

The elements of the output {@link PCollection} which will follow UNION_ALL semantics. Output + * is calculated as follows: Given there are m elements on pipeline which is constructed {@link + * PCollection} (left) and n elements on in provided {@link PCollection} (right): - it will output + * m elements of left and m elements of right. + * + *

Note that this transform requires that the {@code Coder} of the all inputs {@code + * PCollection} to be deterministic (see {@link Coder#verifyDeterministic()}). If the + * collection {@code Coder} is not deterministic, an exception is thrown at pipeline construction + * time. + * + *

All inputs must have equal {@link WindowFn}s and compatible triggers (see {@link + * Trigger#isCompatible(Trigger)}).Triggers with multiple firings may lead to nondeterministic + * results since the this {@code PTransform} is only computed over each individual firing. + * + *

By default, the output {@code PCollection} encodes its elements using the same {@code + * Coder} as that of the first {@code PCollection} in {@code PCollectionList}. + * + *

{@code
+   * Pipeline p = ...;
+   * PCollection first = p.apply(Create.of("1", "1", "2"));
+   * PCollection second = p.apply(Create.of("1", "3", "4", "4"));
+   * PCollection third = p.apply(Create.of("1", "5"));
+   *
+   * // Following example will perform (first intersect second) intersect third.
+   * PCollection results =
+   *     PCollectionList.of(first).and(second).and(third)
+   *     .apply(SetFns.unionAll()); // results will be PCollection containing: "1","1","1","1","2","3","4","4","5"
+   *
+   * }
+ * + * @param the type of the elements in the input {@code PCollectionList} and output {@code + * PCollection}s. + */ + public static Flatten.PCollections unionAll() { + return Flatten.pCollections(); + } + + private static class SetImpl extends PTransform, PCollection> { + + private final transient PCollection rightCollection; + private final PTransform, PCollection> listTransformFn; + + private SetImpl( + PCollection rightCollection, + PTransform, PCollection> listTransformFn) { + this.rightCollection = rightCollection; + this.listTransformFn = listTransformFn; + } + + @Override + public PCollection expand(PCollection leftCollection) { + return PCollectionList.of(leftCollection).and(rightCollection).apply(listTransformFn); + } + } + + private static class SetImplCollections + extends PTransform, PCollection> { + + private final SerializableBiFunction fn; + + private SetImplCollections(SerializableBiFunction fn) { + this.fn = fn; + } + + @Override + public PCollection expand(PCollectionList input) { + List> all = input.getAll(); + MapElements> elementToVoid = + MapElements.via( + new SimpleFunction>() { + @Override + public KV apply(T element) { + return KV.of(element, null); + } + }); + + checkArgument(all.size() > 1, "must have at least two input to a PCollectionList"); + + PCollection first = all.get(0); + Pipeline pipeline = first.getPipeline(); + String firstName = first.getName(); + + List> allTags = new ArrayList<>(); + KeyedPCollectionTuple keyedPCollectionTuple = KeyedPCollectionTuple.empty(pipeline); + + for (PCollection col : all) { + TupleTag tag = new TupleTag<>(); + + PCollection> kvOfElementAndVoid = + col.apply("PrepareKVs" + col.getName(), elementToVoid); + + allTags.add(tag); + keyedPCollectionTuple = keyedPCollectionTuple.and(tag, kvOfElementAndVoid); + } + + PCollection> coGbkResults = + keyedPCollectionTuple.apply("CBKAll" + firstName, CoGroupByKey.create()); + + // TODO: lift combiners through the CoGBK. + PCollection results = + coGbkResults.apply( + "FilterSetElement" + firstName, + ParDo.of( + new DoFn, T>() { + + @ProcessElement + public void processElement(ProcessContext c) { + KV elementGroups = c.element(); + CoGbkResult value = elementGroups.getValue(); + T element = elementGroups.getKey(); + + long numberOfOutputs = Iterables.size(value.getAll(allTags.get(0))); + List> tail = allTags.subList(1, allTags.size()); + + for (TupleTag tag : tail) { + long nextSize = Iterables.size(value.getAll(tag)); + numberOfOutputs = fn.apply(numberOfOutputs, nextSize); + } + for (long i = 0L; i < numberOfOutputs; i++) { + c.output(element); + } + } + })); + + return results.setCoder(first.getCoder()); + } + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java index f8d4a365f8b1..40d2ca9c6332 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java @@ -241,6 +241,10 @@ interface ArgumentProvider { */ TimerMap timerFamily(String tagId); + /** + * Returns the timer id for the current timer of a {@link + * org.apache.beam.sdk.transforms.DoFn.TimerFamily}. + */ String timerId(DoFn doFn); } @@ -532,8 +536,8 @@ public Timer timer(String timerId) { } @Override - public TimerMap timerFamily(String tagId) { - return delegate.timerFamily(tagId); + public TimerMap timerFamily(String timerFamilyId) { + return delegate.timerFamily(timerFamilyId); } @Override diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java new file mode 100644 index 000000000000..a74d72b72b3d --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.transforms.splittabledofn; + +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; + +import java.math.BigDecimal; +import java.math.MathContext; +import org.apache.beam.sdk.annotations.Experimental; +import org.apache.beam.sdk.annotations.Experimental.Kind; +import org.apache.beam.sdk.io.range.OffsetRange; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; + +/** + * An {@link OffsetRangeTracker} for tracking a growable offset range. {@code Long.MAX_VALUE} is + * used as the end of the range to indicate infinity. + * + *

An offset range is considered growable when the end offset could grow (or change) during + * execution time (e.g., Kafka topic partition offset, appended file, ...). + * + *

The growable range is marked as done by claiming {@code Long.MAX_VALUE}. + */ +@Experimental(Kind.SPLITTABLE_DO_FN) +public class GrowableOffsetRangeTracker extends OffsetRangeTracker { + /** + * Provides the estimated end offset of the range. + * + *

{@link #estimate} is called to give the end offset when {@link #trySplit} or {@link + * #getProgress} is invoked. The end offset is exclusive for the range. The estimated end is not + * required to monotonically increase as it will only be taken into consideration when the + * estimated end offset is larger than the current position. Returning {@code Long.MAX_VALUE} as + * the estimate implies the largest possible position for the range is {@code Long.MAX_VALUE - 1}. + * Return {@code Long.MIN_VALUE} if an estimate can not be provided. + * + *

Providing a good estimate is important for an accurate progress signal and will impact + * splitting decisions by the runner. + * + *

If {@link #estimate} is expensive to compute, consider wrapping the implementation with + * {@link Suppliers#memoizeWithExpiration} or equivalent as an optimization. + * + *

TODO(BEAM-10032): Also consider using {@link RangeEndEstimator} when the range is not ended + * with {@code Long.MAX_VALUE}. + */ + @FunctionalInterface + public interface RangeEndEstimator { + long estimate(); + } + + private final RangeEndEstimator rangeEndEstimator; + + public GrowableOffsetRangeTracker(long start, RangeEndEstimator rangeEndEstimator) { + super(new OffsetRange(start, Long.MAX_VALUE)); + this.rangeEndEstimator = checkNotNull(rangeEndEstimator); + } + + @Override + public SplitResult trySplit(double fractionOfRemainder) { + // If current tracking range is no longer growable, split it as a normal range. + if (range.getTo() != Long.MAX_VALUE || range.getTo() == range.getFrom()) { + return super.trySplit(fractionOfRemainder); + } + // If current range has been done, there is no more space to split. + if (lastAttemptedOffset != null && lastAttemptedOffset == Long.MAX_VALUE) { + return null; + } + BigDecimal cur = + (lastAttemptedOffset == null) + ? BigDecimal.valueOf(range.getFrom()).subtract(BigDecimal.ONE, MathContext.DECIMAL128) + : BigDecimal.valueOf(lastAttemptedOffset); + + // Fetch the estimated end offset. If the estimated end is smaller than the next offset, use + // the next offset as end. + BigDecimal estimateRangeEnd = + BigDecimal.valueOf(rangeEndEstimator.estimate()) + .max(cur.add(BigDecimal.ONE, MathContext.DECIMAL128)); + + // Convert to BigDecimal in computation to prevent overflow, which may result in loss of + // precision. + // split = cur + max(1, (estimateRangeEnd - cur) * fractionOfRemainder) + BigDecimal splitPos = + cur.add( + estimateRangeEnd + .subtract(cur, MathContext.DECIMAL128) + .multiply(BigDecimal.valueOf(fractionOfRemainder), MathContext.DECIMAL128) + .max(BigDecimal.ONE), + MathContext.DECIMAL128); + long split = splitPos.longValue(); + if (split > estimateRangeEnd.longValue()) { + return null; + } + OffsetRange res = new OffsetRange(split, range.getTo()); + this.range = new OffsetRange(range.getFrom(), split); + return SplitResult.of(range, res); + } + + @Override + public Progress getProgress() { + // If current tracking range is no longer growable, get progress as a normal range. + if (range.getTo() != Long.MAX_VALUE || range.getTo() == range.getFrom()) { + return super.getProgress(); + } + + // Convert to BigDecimal in computation to prevent overflow, which may result in lost of + // precision. + BigDecimal estimateRangeEnd = BigDecimal.valueOf(rangeEndEstimator.estimate()); + + if (lastAttemptedOffset == null) { + return Progress.from( + 0, + estimateRangeEnd + .subtract(BigDecimal.valueOf(range.getFrom()), MathContext.DECIMAL128) + .max(BigDecimal.ZERO) + .doubleValue()); + } + + BigDecimal workRemaining = + estimateRangeEnd + .subtract(BigDecimal.valueOf(lastAttemptedOffset), MathContext.DECIMAL128) + .max(BigDecimal.ZERO); + BigDecimal totalWork = + estimateRangeEnd + .max(BigDecimal.valueOf(lastAttemptedOffset)) + .subtract(BigDecimal.valueOf(range.getFrom()), MathContext.DECIMAL128); + return Progress.from( + totalWork.subtract(workRemaining, MathContext.DECIMAL128).doubleValue(), + workRemaining.doubleValue()); + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java index 2743a1dc56c4..9c7116f5e8c1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java @@ -21,6 +21,8 @@ import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import java.math.BigDecimal; +import java.math.MathContext; import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.annotations.Experimental.Kind; @@ -31,13 +33,16 @@ /** * A {@link RestrictionTracker} for claiming offsets in an {@link OffsetRange} in a monotonically * increasing fashion. + * + *

The smallest offset is {@code Long.MIN_VALUE} and the largest offset is {@code Long.MAX_VALUE + * - 1}. */ @Experimental(Kind.SPLITTABLE_DO_FN) public class OffsetRangeTracker extends RestrictionTracker implements HasProgress { - private OffsetRange range; - @Nullable private Long lastClaimedOffset = null; - @Nullable private Long lastAttemptedOffset = null; + protected OffsetRange range; + @Nullable protected Long lastClaimedOffset = null; + @Nullable protected Long lastAttemptedOffset = null; public OffsetRangeTracker(OffsetRange range) { this.range = checkNotNull(range); @@ -50,16 +55,27 @@ public OffsetRange currentRestriction() { @Override public SplitResult trySplit(double fractionOfRemainder) { - long cur = (lastAttemptedOffset == null) ? range.getFrom() - 1 : lastAttemptedOffset; - long splitPos = - cur - + Math.max( - 1L, (Double.valueOf((range.getTo() - cur) * fractionOfRemainder)).longValue()); - if (splitPos >= range.getTo()) { + // Convert to BigDecimal in computation to prevent overflow, which may result in loss of + // precision. + BigDecimal cur = + (lastAttemptedOffset == null) + ? BigDecimal.valueOf(range.getFrom()).subtract(BigDecimal.ONE, MathContext.DECIMAL128) + : BigDecimal.valueOf(lastAttemptedOffset); + // split = cur + max(1, (range.getTo() - cur) * fractionOfRemainder) + BigDecimal splitPos = + cur.add( + BigDecimal.valueOf(range.getTo()) + .subtract(cur, MathContext.DECIMAL128) + .multiply(BigDecimal.valueOf(fractionOfRemainder), MathContext.DECIMAL128) + .max(BigDecimal.ONE), + MathContext.DECIMAL128); + + long split = splitPos.longValue(); + if (split >= range.getTo()) { return null; } - OffsetRange res = new OffsetRange(splitPos, range.getTo()); - this.range = new OffsetRange(range.getFrom(), splitPos); + OffsetRange res = new OffsetRange(split, range.getTo()); + this.range = new OffsetRange(range.getFrom(), split); return SplitResult.of(range, res); } @@ -120,13 +136,27 @@ public String toString() { public Progress getProgress() { // If we have never attempted an offset, we return the length of the entire range as work // remaining. + // Convert to BigDecimal in computation to prevent overflow, which may result in loss of + // precision. if (lastAttemptedOffset == null) { - return Progress.from(0, range.getTo() - range.getFrom()); + return Progress.from( + 0, + BigDecimal.valueOf(range.getTo()) + .subtract(BigDecimal.valueOf(range.getFrom()), MathContext.DECIMAL128) + .doubleValue()); } // Compute the amount of work remaining from where we are to where we are attempting to get to // with a minimum of zero in case we have claimed beyond the end of the range. - long workRemaining = Math.max(range.getTo() - lastAttemptedOffset, 0); - return Progress.from(range.getTo() - range.getFrom() - workRemaining, workRemaining); + BigDecimal workRemaining = + BigDecimal.valueOf(range.getTo()) + .subtract(BigDecimal.valueOf(lastAttemptedOffset), MathContext.DECIMAL128) + .max(BigDecimal.ZERO); + BigDecimal totalWork = + BigDecimal.valueOf(range.getTo()) + .subtract(BigDecimal.valueOf(range.getFrom()), MathContext.DECIMAL128); + return Progress.from( + totalWork.subtract(workRemaining, MathContext.DECIMAL128).doubleValue(), + workRemaining.doubleValue()); } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java index 15aab64787ad..3470dbfd55bd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java @@ -79,6 +79,7 @@ public class BufferedElementCountingOutputStream extends OutputStream { public static final int DEFAULT_BUFFER_SIZE = 64 * 1024; private final ByteBuffer buffer; private final OutputStream os; + private final long terminatorValue; private boolean finished; private long count; @@ -87,15 +88,20 @@ public class BufferedElementCountingOutputStream extends OutputStream { * manner. */ public BufferedElementCountingOutputStream(OutputStream os) { - this(os, DEFAULT_BUFFER_SIZE); + this(os, DEFAULT_BUFFER_SIZE, 0); + } + + public BufferedElementCountingOutputStream(OutputStream os, long terminatorValue) { + this(os, DEFAULT_BUFFER_SIZE, terminatorValue); } /** * Creates an output stream which encodes the number of elements output to it in a streaming * manner with the given {@code bufferSize}. */ - BufferedElementCountingOutputStream(OutputStream os, int bufferSize) { + BufferedElementCountingOutputStream(OutputStream os, int bufferSize, long terminatorValue) { this.os = os; + this.terminatorValue = terminatorValue; this.finished = false; this.count = 0; ByteBuffer buffer = BUFFER_POOL.poll(); @@ -111,8 +117,8 @@ public void finish() throws IOException { return; } flush(); - // Finish the stream by stating that there are 0 elements that follow. - VarInt.encode(0, os); + // Finish the stream with the terminatorValue. + VarInt.encode(terminatorValue, os); if (!BUFFER_POOL.offer(buffer)) { // The pool is full, we can't store the buffer. We just drop the buffer. } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java new file mode 100644 index 000000000000..ed07a65b0219 --- /dev/null +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.transforms; + +import static junit.framework.TestCase.assertEquals; + +import java.util.Arrays; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.NeedsRunner; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SetsTest { + + @Rule public final TestPipeline p = TestPipeline.create(); + + Schema schema = Schema.builder().addStringField("alphabet").build(); + + static PCollection first; + static PCollection second; + static PCollection firstRows; + static PCollection secondRows; + + private Iterable toRows(String... values) { + return Iterables.transform( + Arrays.asList(values), (elem) -> Row.withSchema(schema).addValues(elem).build()); + } + + @Before + public void setup() { + final String[] firstData = {"a", "a", "a", "b", "b", "c", "d", "d", "g", "g", "h", "h"}; + final String[] secondData = {"a", "a", "b", "b", "b", "c", "d", "d", "e", "e", "f", "f"}; + + first = p.apply("first", Create.of(Arrays.asList(firstData))); + second = p.apply("second", Create.of(Arrays.asList(secondData))); + + firstRows = p.apply("firstRows", Create.of(toRows(firstData)).withRowSchema(schema)); + secondRows = p.apply("secondRows", Create.of(toRows(secondData)).withRowSchema(schema)); + } + + @Test + @Category(NeedsRunner.class) + public void testIntersection() { + PAssert.that(first.apply("strings", Sets.intersectDistinct(second))) + .containsInAnyOrder("a", "b", "c", "d"); + + PCollection results = firstRows.apply("rows", Sets.intersectDistinct(secondRows)); + + PAssert.that(results).containsInAnyOrder(toRows("a", "b", "c", "d")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testIntersectionCollectionList() { + + PCollection third = p.apply("third", Create.of(Arrays.asList("b", "b", "c", "f"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("b", "b", "c", "f"))); + + PAssert.that( + PCollectionList.of(first) + .and(second) + .and(third) + .apply("stringsCols", Sets.intersectDistinct())) + .containsInAnyOrder("b", "c"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.intersectDistinct()); + + PAssert.that(results).containsInAnyOrder(toRows("b", "c")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testIntersectionAll() { + + PAssert.that(first.apply("strings", Sets.intersectAll(second))) + .containsInAnyOrder("a", "a", "b", "b", "c", "d", "d"); + + PCollection results = firstRows.apply("rows", Sets.intersectAll(secondRows)); + + PAssert.that(results).containsInAnyOrder(toRows("a", "a", "b", "b", "c", "d", "d")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testIntersectionAllCollectionList() { + PCollection third = p.apply("third", Create.of(Arrays.asList("a", "b", "f"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("a", "b", "f"))); + + PAssert.that( + PCollectionList.of(first) + .and(second) + .and(third) + .apply("stringsCols", Sets.intersectAll())) + .containsInAnyOrder("a", "b"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.intersectAll()); + + PAssert.that(results).containsInAnyOrder(toRows("a", "b")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testExcept() { + + PAssert.that(first.apply("strings", Sets.exceptDistinct(second))).containsInAnyOrder("g", "h"); + + PCollection results = firstRows.apply("rows", Sets.exceptDistinct(secondRows)); + + PAssert.that(results).containsInAnyOrder(toRows("g", "h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testExceptCollectionList() { + PCollection third = p.apply("third", Create.of(Arrays.asList("a", "b", "b", "g", "g"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("a", "b", "b", "g", "g"))); + + PAssert.that( + PCollectionList.of(first) + .and(second) + .and(third) + .apply("stringsCols", Sets.exceptDistinct())) + .containsInAnyOrder("h"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.exceptDistinct()); + + PAssert.that(results).containsInAnyOrder(toRows("h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testExceptAll() { + + PAssert.that(first.apply("strings", Sets.exceptAll(second))) + .containsInAnyOrder("a", "g", "g", "h", "h"); + + PCollection results = firstRows.apply("rows", Sets.exceptAll(secondRows)); + + PAssert.that(results).containsInAnyOrder(toRows("a", "g", "g", "h", "h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testExceptAllCollectionList() { + PCollection third = p.apply("third", Create.of(Arrays.asList("a", "b", "b", "g", "f"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("a", "b", "b", "g"))); + + PAssert.that( + PCollectionList.of(first).and(second).and(third).apply("stringsCols", Sets.exceptAll())) + .containsInAnyOrder("g", "h", "h"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.exceptAll()); + + PAssert.that(results).containsInAnyOrder(toRows("g", "h", "h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testUnion() { + + PAssert.that(first.apply("strings", Sets.unionDistinct(second))) + .containsInAnyOrder("a", "b", "c", "d", "e", "f", "g", "h"); + + PCollection results = firstRows.apply("rows", Sets.unionDistinct(secondRows)); + + PAssert.that(results).containsInAnyOrder(toRows("a", "b", "c", "d", "e", "f", "g", "h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testUnionCollectionList() { + PCollection third = p.apply("third", Create.of(Arrays.asList("a", "k", "k"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("a", "k", "k"))); + + PAssert.that( + PCollectionList.of(first) + .and(second) + .and(third) + .apply("stringsCols", Sets.unionDistinct())) + .containsInAnyOrder("a", "b", "c", "d", "e", "f", "g", "h", "k"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.unionDistinct()); + + PAssert.that(results).containsInAnyOrder(toRows("a", "b", "c", "d", "e", "f", "g", "h", "k")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testUnionAll() { + + PAssert.that(first.apply("strings", Sets.unionAll(second))) + .containsInAnyOrder( + "a", "a", "a", "a", "a", "b", "b", "b", "b", "b", "c", "c", "d", "d", "d", "d", "e", + "e", "f", "f", "g", "g", "h", "h"); + + PCollection results = firstRows.apply("rows", Sets.unionAll(secondRows)); + + PAssert.that(results) + .containsInAnyOrder( + toRows( + "a", "a", "a", "a", "a", "b", "b", "b", "b", "b", "c", "c", "d", "d", "d", "d", "e", + "e", "f", "f", "g", "g", "h", "h")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testUnionAllCollections() { + + PCollection third = p.apply("third", Create.of(Arrays.asList("a", "b", "b", "k", "k"))); + PCollection thirdRows = p.apply("thirdRows", Create.of(toRows("a", "b", "b", "k", "k"))); + + PAssert.that( + PCollectionList.of(first).and(second).and(third).apply("stringsCols", Sets.unionAll())) + .containsInAnyOrder( + "a", "a", "a", "a", "a", "a", "b", "b", "b", "b", "b", "b", "b", "c", "c", "d", "d", + "d", "d", "e", "e", "f", "f", "g", "g", "h", "h", "k", "k"); + + PCollection results = + PCollectionList.of(firstRows) + .and(secondRows) + .and(thirdRows) + .apply("rowCols", Sets.unionAll()); + + PAssert.that(results) + .containsInAnyOrder( + toRows( + "a", "a", "a", "a", "a", "a", "b", "b", "b", "b", "b", "b", "b", "c", "c", "d", "d", + "d", "d", "e", "e", "f", "f", "g", "g", "h", "h", "k", "k")); + + assertEquals(schema, results.getSchema()); + + p.run(); + } +} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTrackerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTrackerTest.java new file mode 100644 index 000000000000..17050157d4cc --- /dev/null +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTrackerTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.transforms.splittabledofn; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.math.BigDecimal; +import java.math.MathContext; +import org.apache.beam.sdk.io.range.OffsetRange; +import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.Progress; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link GrowableOffsetRangeTracker}. */ +@RunWith(JUnit4.class) +public class GrowableOffsetRangeTrackerTest { + private static class SimpleEstimator implements GrowableOffsetRangeTracker.RangeEndEstimator { + private long estimateRangeEnd = 0; + + @Override + public long estimate() { + return estimateRangeEnd; + } + + public void setEstimateRangeEnd(long offset) { + estimateRangeEnd = offset; + } + } + + @Rule public final ExpectedException expected = ExpectedException.none(); + + @Test + public void testIllegalInitialization() throws Exception { + expected.expect(NullPointerException.class); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, null); + } + + @Test + public void testTryClaim() throws Exception { + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, new SimpleEstimator()); + assertTrue(tracker.tryClaim(10L)); + assertTrue(tracker.tryClaim(100L)); + assertFalse(tracker.tryClaim(Long.MAX_VALUE)); + tracker.checkDone(); + } + + @Test + public void testCheckpointBeforeStart() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + simpleEstimator.setEstimateRangeEnd(10); + SplitResult res = tracker.trySplit(0); + tracker.checkDone(); + assertEquals(new OffsetRange(0, 0), res.getPrimary()); + assertEquals(new OffsetRange(0, 0), tracker.currentRestriction()); + assertEquals(new OffsetRange(0, Long.MAX_VALUE), res.getResidual()); + } + + @Test + public void testCheckpointJustStarted() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + assertTrue(tracker.tryClaim(5L)); + simpleEstimator.setEstimateRangeEnd(0L); + SplitResult res = tracker.trySplit(0); + tracker.checkDone(); + assertEquals(new OffsetRange(0, 6), res.getPrimary()); + assertEquals(new OffsetRange(0, 6), tracker.currentRestriction()); + assertEquals(new OffsetRange(6, Long.MAX_VALUE), res.getResidual()); + + tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + assertTrue(tracker.tryClaim(5L)); + simpleEstimator.setEstimateRangeEnd(20L); + res = tracker.trySplit(0); + tracker.checkDone(); + assertEquals(new OffsetRange(0, 6), res.getPrimary()); + assertEquals(new OffsetRange(6, Long.MAX_VALUE), res.getResidual()); + } + + @Test + public void testCheckpointAfterAllProcessed() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + assertFalse(tracker.tryClaim(Long.MAX_VALUE)); + tracker.checkDone(); + assertNull(tracker.trySplit(0)); + } + + @Test + public void testCheckpointAtEmptyRange() throws Exception { + GrowableOffsetRangeTracker tracker = + new GrowableOffsetRangeTracker(Long.MAX_VALUE, new SimpleEstimator()); + tracker.checkDone(); + assertNull(tracker.trySplit(0)); + } + + @Test + public void testSplit() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + assertTrue(tracker.tryClaim(0L)); + + simpleEstimator.setEstimateRangeEnd(16L); + // The split of infinite range results in one finite range and one infinite range. + SplitResult res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(0, 8), res.getPrimary()); + assertEquals(new OffsetRange(0, 8), tracker.currentRestriction()); + assertEquals(new OffsetRange(8, Long.MAX_VALUE), res.getResidual()); + + // After the first split, the tracker should track a finite range. Estimate offset should not + // impact split. + simpleEstimator.setEstimateRangeEnd(12L); + res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(0, 4), res.getPrimary()); + assertEquals(new OffsetRange(0, 4), tracker.currentRestriction()); + assertEquals(new OffsetRange(4, 8), res.getResidual()); + assertFalse(tracker.tryClaim(4L)); + tracker.checkDone(); + } + + @Test + public void testSplitWithMaxEstimateRangeEnd() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(0L, simpleEstimator); + assertTrue(tracker.tryClaim(1L)); + simpleEstimator.setEstimateRangeEnd(Long.MAX_VALUE); + SplitResult res = tracker.trySplit(0.5); + long expectedEnd = 1L + (Long.MAX_VALUE - 1L) / 2; + assertEquals(new OffsetRange(0L, expectedEnd), res.getPrimary()); + assertEquals(new OffsetRange(expectedEnd, Long.MAX_VALUE), res.getResidual()); + } + + @Test + public void testProgressBeforeStart() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(10L, simpleEstimator); + simpleEstimator.setEstimateRangeEnd(20); + Progress currentProcess = tracker.getProgress(); + assertEquals(0, currentProcess.getWorkCompleted(), 0.001); + assertEquals(10, currentProcess.getWorkRemaining(), 0.001); + + simpleEstimator.setEstimateRangeEnd(15); + currentProcess = tracker.getProgress(); + assertEquals(0, currentProcess.getWorkCompleted(), 0.001); + assertEquals(5, currentProcess.getWorkRemaining(), 0.001); + + simpleEstimator.setEstimateRangeEnd(5); + currentProcess = tracker.getProgress(); + assertEquals(0, currentProcess.getWorkCompleted(), 0.001); + assertEquals(0, currentProcess.getWorkRemaining(), 0.001); + } + + @Test + public void testProgressAfterFinished() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(10L, simpleEstimator); + assertFalse(tracker.tryClaim(Long.MAX_VALUE)); + tracker.checkDone(); + simpleEstimator.setEstimateRangeEnd(0L); + Progress currentProgress = tracker.getProgress(); + assertEquals(Long.MAX_VALUE - 10L, currentProgress.getWorkCompleted(), 0.001); + assertEquals(0, currentProgress.getWorkRemaining(), 0.001); + } + + @Test + public void testProgress() throws Exception { + long start = 10L; + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = new GrowableOffsetRangeTracker(start, simpleEstimator); + long cur = 20L; + assertTrue(tracker.tryClaim(cur)); + + simpleEstimator.setEstimateRangeEnd(5L); + Progress currentProgress = tracker.getProgress(); + assertEquals(cur - start, currentProgress.getWorkCompleted(), 0.001); + assertEquals(0, currentProgress.getWorkRemaining(), 0.001); + + simpleEstimator.setEstimateRangeEnd(35L); + currentProgress = tracker.getProgress(); + assertEquals(cur - start, currentProgress.getWorkCompleted(), 0.001); + assertEquals(35L - cur, currentProgress.getWorkRemaining(), 0.001); + + simpleEstimator.setEstimateRangeEnd(25L); + currentProgress = tracker.getProgress(); + assertEquals(cur - start, currentProgress.getWorkCompleted(), 0.001); + assertEquals(25L - cur, currentProgress.getWorkRemaining(), 0.001); + + simpleEstimator.setEstimateRangeEnd(Long.MAX_VALUE); + currentProgress = tracker.getProgress(); + assertEquals(cur - start, currentProgress.getWorkCompleted(), 0.001); + assertEquals(Long.MAX_VALUE - cur, currentProgress.getWorkRemaining(), 0.001); + } + + @Test + public void testLargeRange() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = + new GrowableOffsetRangeTracker(Long.MIN_VALUE, simpleEstimator); + + simpleEstimator.setEstimateRangeEnd(Long.MAX_VALUE); + Progress progress = tracker.getProgress(); + assertEquals(0, progress.getWorkCompleted(), 0.001); + assertEquals( + BigDecimal.valueOf(Long.MAX_VALUE) + .subtract(BigDecimal.valueOf(Long.MIN_VALUE), MathContext.DECIMAL128) + .doubleValue(), + progress.getWorkRemaining(), + 0.001); + + simpleEstimator.setEstimateRangeEnd(Long.MIN_VALUE); + SplitResult res = tracker.trySplit(0); + assertEquals(new OffsetRange(Long.MIN_VALUE, Long.MIN_VALUE), res.getPrimary()); + assertEquals(new OffsetRange(Long.MIN_VALUE, Long.MAX_VALUE), res.getResidual()); + } + + @Test + public void testSmallRangeWithLargeValue() throws Exception { + SimpleEstimator simpleEstimator = new SimpleEstimator(); + GrowableOffsetRangeTracker tracker = + new GrowableOffsetRangeTracker(123456789012345677L, simpleEstimator); + assertTrue(tracker.tryClaim(123456789012345677L)); + simpleEstimator.setEstimateRangeEnd(123456789012345679L); + SplitResult res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(123456789012345677L, 123456789012345678L), res.getPrimary()); + assertEquals(new OffsetRange(123456789012345678L, Long.MAX_VALUE), res.getResidual()); + + tracker = new GrowableOffsetRangeTracker(123456789012345681L, simpleEstimator); + assertTrue(tracker.tryClaim(123456789012345681L)); + simpleEstimator.setEstimateRangeEnd(123456789012345683L); + res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(123456789012345681L, 123456789012345682L), res.getPrimary()); + assertEquals(new OffsetRange(123456789012345682L, Long.MAX_VALUE), res.getResidual()); + } +} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java index 6fdd2549a301..b2c381c58590 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.math.BigDecimal; +import java.math.MathContext; import org.apache.beam.sdk.io.range.OffsetRange; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.Progress; import org.junit.Rule; @@ -35,6 +37,12 @@ public class OffsetRangeTrackerTest { @Rule public final ExpectedException expected = ExpectedException.none(); + @Test + public void testIllegalInitialization() throws Exception { + expected.expect(NullPointerException.class); + OffsetRangeTracker tracker = new OffsetRangeTracker(null); + } + @Test public void testTryClaim() throws Exception { OffsetRange range = new OffsetRange(100, 200); @@ -231,4 +239,39 @@ public void testBacklogPartiallyCompleted() { assertEquals(50, progress.getWorkCompleted(), 0.001); assertEquals(50, progress.getWorkRemaining(), 0.001); } + + @Test + public void testLargeRange() throws Exception { + OffsetRangeTracker tracker = + new OffsetRangeTracker(new OffsetRange(Long.MIN_VALUE, Long.MAX_VALUE)); + + Progress progress = tracker.getProgress(); + assertEquals(0, progress.getWorkCompleted(), 0.001); + assertEquals( + BigDecimal.valueOf(Long.MAX_VALUE) + .subtract(BigDecimal.valueOf(Long.MIN_VALUE), MathContext.DECIMAL128) + .doubleValue(), + progress.getWorkRemaining(), + 0.001); + + SplitResult res = tracker.trySplit(0); + assertEquals(new OffsetRange(Long.MIN_VALUE, Long.MIN_VALUE), res.getPrimary()); + assertEquals(new OffsetRange(Long.MIN_VALUE, Long.MAX_VALUE), res.getResidual()); + } + + @Test + public void testSmallRangeWithLargeValue() throws Exception { + OffsetRangeTracker tracker = + new OffsetRangeTracker(new OffsetRange(123456789012345677L, 123456789012345679L)); + assertTrue(tracker.tryClaim(123456789012345677L)); + SplitResult res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(123456789012345677L, 123456789012345678L), res.getPrimary()); + assertEquals(new OffsetRange(123456789012345678L, 123456789012345679L), res.getResidual()); + + tracker = new OffsetRangeTracker(new OffsetRange(123456789012345681L, 123456789012345683L)); + assertTrue(tracker.tryClaim(123456789012345681L)); + res = tracker.trySplit(0.5); + assertEquals(new OffsetRange(123456789012345681L, 123456789012345682L), res.getPrimary()); + assertEquals(new OffsetRange(123456789012345682L, 123456789012345683L), res.getResidual()); + } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java index 205f1afe1e52..0d1f7d89b719 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java @@ -41,14 +41,22 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import org.junit.runners.Parameterized; /** Tests for {@link BufferedElementCountingOutputStream}. */ -@RunWith(JUnit4.class) +@RunWith(Parameterized.class) public class BufferedElementCountingOutputStreamTest { @Rule public final ExpectedException expectedException = ExpectedException.none(); private static final int BUFFER_SIZE = 8; + @Parameterized.Parameters + public static Iterable data() { + return ImmutableList.builder().add(new Object[] {0L}).add(new Object[] {-1L}).build(); + } + + @Parameterized.Parameter(0) + public long terminatorValue; + @Test public void testEmptyValues() throws Exception { testValues(Collections.emptyList()); @@ -219,6 +227,8 @@ private void verifyValues(List expectedValues, InputStream is) throws Ex } } while (count > 0); + assertEquals(terminatorValue, count); + if (expectedValues.isEmpty()) { assertTrue(values.isEmpty()); } else { @@ -229,7 +239,7 @@ private void verifyValues(List expectedValues, InputStream is) throws Ex private BufferedElementCountingOutputStream createAndWriteValues( List values, OutputStream output) throws Exception { BufferedElementCountingOutputStream os = - new BufferedElementCountingOutputStream(output, BUFFER_SIZE); + new BufferedElementCountingOutputStream(output, BUFFER_SIZE, terminatorValue); for (byte[] value : values) { os.markElementStart(); diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java index 578e87583438..f5707eaafca0 100644 --- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java @@ -39,6 +39,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.runners.core.construction.BeamUrns; import org.apache.beam.runners.core.construction.CoderTranslation; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.runners.core.construction.Environments; import org.apache.beam.runners.core.construction.PipelineTranslation; import org.apache.beam.runners.core.construction.RehydratedComponents; @@ -203,7 +204,7 @@ private static Coder resolveCoder(List coderUrns) throws Exception { RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(componentsBuilder.build()); - return CoderTranslation.fromProto(coder, rehydratedComponents); + return CoderTranslation.fromProto(coder, rehydratedComponents, TranslationContext.DEFAULT); } private static RunnerApi.Coder buildProto( diff --git a/sdks/java/extensions/ml/build.gradle b/sdks/java/extensions/ml/build.gradle index 1f61c6a6da4a..c5e1ed3905fc 100644 --- a/sdks/java/extensions/ml/build.gradle +++ b/sdks/java/extensions/ml/build.gradle @@ -1,3 +1,5 @@ +import groovy.json.JsonOutput + /* * * * Licensed to the Apache Software Foundation (ASF) under one @@ -19,7 +21,7 @@ */ plugins { id 'org.apache.beam.module' } -applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.protobuf') +applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.ml') description = 'Apache Beam :: SDKs :: Java :: Extensions :: ML' @@ -27,16 +29,25 @@ dependencies { compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:expansion-service") compile 'com.google.cloud:google-cloud-video-intelligence:1.2.0' + compile 'com.google.cloud:google-cloud-dlp:1.1.4' compile 'com.google.cloud:google-cloud-language:1.99.4' + provided library.java.junit + testCompile project(path: ':sdks:java:core', configuration: 'shadowTest') testCompile project(path: ':sdks:java:core', configuration: 'shadowTest') testCompile library.java.mockito_core testCompile 'com.google.cloud:google-cloud-video-intelligence:1.2.0' + testCompile 'com.google.cloud:google-cloud-dlp:1.1.4' + testCompile project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntime") testCompile 'com.google.cloud:google-cloud-language:1.99.4' - testCompile library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") testRuntimeOnly project(":runners:google-cloud-dataflow-java") } project.test { + def gcpProject = project.findProperty("gcpProject") ?: 'apache-beam-testing' include "**/**IT.class" + def pipelineOptions = [ + "--project=${gcpProject}" + ] + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions) } diff --git a/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDLP.java b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDLP.java new file mode 100644 index 000000000000..aabac4e0fcda --- /dev/null +++ b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDLP.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.privacy.dlp.v2.Table; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.beam.sdk.annotations.Experimental; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.StateSpec; +import org.apache.beam.sdk.state.StateSpecs; +import org.apache.beam.sdk.state.TimeDomain; +import org.apache.beam.sdk.state.Timer; +import org.apache.beam.sdk.state.TimerSpec; +import org.apache.beam.sdk.state.TimerSpecs; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.values.KV; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Batches input rows to reduce number of requests sent to Cloud DLP service. */ +@Experimental +class BatchRequestForDLP extends DoFn, KV>> { + public static final Logger LOG = LoggerFactory.getLogger(BatchRequestForDLP.class); + + private final Counter numberOfRowsBagged = + Metrics.counter(BatchRequestForDLP.class, "numberOfRowsBagged"); + + private final Integer batchSizeBytes; + + @StateId("elementsBag") + private final StateSpec>> elementsBag = StateSpecs.bag(); + + @TimerId("eventTimer") + private final TimerSpec eventTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME); + + /** + * Constructs the batching DoFn. + * + * @param batchSize Desired batch size in bytes. + */ + public BatchRequestForDLP(Integer batchSize) { + this.batchSizeBytes = batchSize; + } + + @ProcessElement + public void process( + @Element KV element, + @StateId("elementsBag") BagState> elementsBag, + @TimerId("eventTimer") Timer eventTimer, + BoundedWindow w) { + elementsBag.add(element); + eventTimer.set(w.maxTimestamp()); + } + + /** + * Outputs the elements buffered in the elementsBag in batches of desired size. + * + * @param elementsBag element buffer. + * @param output Batched input elements. + */ + @OnTimer("eventTimer") + public void onTimer( + @StateId("elementsBag") BagState> elementsBag, + OutputReceiver>> output) { + if (elementsBag.read().iterator().hasNext()) { + String key = elementsBag.read().iterator().next().getKey(); + AtomicInteger bufferSize = new AtomicInteger(); + List rows = new ArrayList<>(); + elementsBag + .read() + .forEach( + element -> { + int elementSize = element.getValue().getSerializedSize(); + boolean clearBuffer = bufferSize.intValue() + elementSize > batchSizeBytes; + if (clearBuffer) { + LOG.debug( + "Clear buffer of {} bytes, Key {}", bufferSize.intValue(), element.getKey()); + numberOfRowsBagged.inc(rows.size()); + output.output(KV.of(element.getKey(), rows)); + rows.clear(); + bufferSize.set(0); + } + rows.add(element.getValue()); + bufferSize.getAndAdd(element.getValue().getSerializedSize()); + }); + if (!rows.isEmpty()) { + LOG.debug("Outputting remaining {} rows.", rows.size()); + numberOfRowsBagged.inc(rows.size()); + output.output(KV.of(key, rows)); + } + } + } +} diff --git a/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyText.java b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyText.java new file mode 100644 index 000000000000..0502950c70f6 --- /dev/null +++ b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyText.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.auto.value.AutoValue; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.ProjectName; +import com.google.privacy.dlp.v2.Table; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.sdk.annotations.Experimental; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionView; + +/** + * A {@link PTransform} connecting to Cloud DLP (https://cloud.google.com/dlp/docs/libraries) and + * deidentifying text according to provided settings. The transform supports both columnar delimited + * input data (eg. CSV) and unstructured input. + * + *

If the headerColumns property is set and a sideinput with table headers is added to the + * PTransform, delimiter also should be set, else the results will be incorrect. If headerColumns is + * neither set nor passed as side input, input is assumed to be unstructured. + * + *

Either deidentifyTemplateName (String) or deidentifyConfig {@link DeidentifyConfig} need to be + * set. inspectTemplateName and inspectConfig ({@link InspectConfig} are optional. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + * + *

The transform consumes {@link KV} of {@link String}s (assumed to be filename as key and + * contents as value) and outputs {@link KV} of {@link String} (eg. filename) and {@link + * DeidentifyContentResponse}, which will contain {@link Table} of results for the user to consume. + */ +@Experimental +@AutoValue +public abstract class DLPDeidentifyText + extends PTransform< + PCollection>, PCollection>> { + + public static final Integer DLP_PAYLOAD_LIMIT_BYTES = 524000; + + /** @return Template name for data inspection. */ + @Nullable + public abstract String getInspectTemplateName(); + + /** @return Template name for data deidentification. */ + @Nullable + public abstract String getDeidentifyTemplateName(); + + /** + * @return Configuration object for data inspection. If present, supersedes the template settings. + */ + @Nullable + public abstract InspectConfig getInspectConfig(); + + /** @return Configuration object for deidentification. If present, supersedes the template. */ + @Nullable + public abstract DeidentifyConfig getDeidentifyConfig(); + + /** @return List of column names if the input KV value is a delimited row. */ + @Nullable + public abstract PCollectionView> getHeaderColumns(); + + /** @return Delimiter to be used when splitting values from input strings into columns. */ + @Nullable + public abstract String getColumnDelimiter(); + + /** @return Size of input elements batch to be sent to Cloud DLP service in one request. */ + public abstract Integer getBatchSizeBytes(); + + /** @return ID of Google Cloud project to be used when deidentifying data. */ + public abstract String getProjectId(); + + @AutoValue.Builder + public abstract static class Builder { + /** @param inspectTemplateName Template name for data inspection. */ + public abstract Builder setInspectTemplateName(String inspectTemplateName); + + /** @param headerColumns List of column names if the input KV value is a delimited row. */ + public abstract Builder setHeaderColumns(PCollectionView> headerColumns); + + /** + * @param delimiter Delimiter to be used when splitting values from input strings into columns. + */ + public abstract Builder setColumnDelimiter(String delimiter); + + /** + * @param batchSize Size of input elements batch to be sent to Cloud DLP service in one request. + */ + public abstract Builder setBatchSizeBytes(Integer batchSize); + + /** @param projectId ID of Google Cloud project to be used when deidentifying data. */ + public abstract Builder setProjectId(String projectId); + + /** @param deidentifyTemplateName Template name for data deidentification. */ + public abstract Builder setDeidentifyTemplateName(String deidentifyTemplateName); + + /** + * @param inspectConfig Configuration object for data inspection. If present, supersedes the + * template settings. + */ + public abstract Builder setInspectConfig(InspectConfig inspectConfig); + + /** + * @param deidentifyConfig Configuration object for data deidentification. If present, + * supersedes the template settings. + */ + public abstract Builder setDeidentifyConfig(DeidentifyConfig deidentifyConfig); + + abstract DLPDeidentifyText autoBuild(); + + public DLPDeidentifyText build() { + DLPDeidentifyText dlpDeidentifyText = autoBuild(); + if (dlpDeidentifyText.getDeidentifyConfig() == null + && dlpDeidentifyText.getDeidentifyTemplateName() == null) { + throw new IllegalArgumentException( + "Either deidentifyConfig or deidentifyTemplateName need to be set!"); + } + if (dlpDeidentifyText.getBatchSizeBytes() > DLP_PAYLOAD_LIMIT_BYTES) { + throw new IllegalArgumentException( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLP_PAYLOAD_LIMIT_BYTES)); + } + if (dlpDeidentifyText.getColumnDelimiter() == null + && dlpDeidentifyText.getHeaderColumns() != null) { + throw new IllegalArgumentException( + "Column delimiter should be set if headers are present."); + } + if (dlpDeidentifyText.getHeaderColumns() == null + && dlpDeidentifyText.getColumnDelimiter() != null) { + throw new IllegalArgumentException( + "Column headers should be supplied when delimiter is present."); + } + return dlpDeidentifyText; + } + } + + public static DLPDeidentifyText.Builder newBuilder() { + return new AutoValue_DLPDeidentifyText.Builder(); + } + + /** + * The transform converts the contents of input PCollection into {@link Table.Row}s and then calls + * Cloud DLP service to perform the deidentification according to provided settings. + * + * @param input input PCollection + * @return PCollection after transformations + */ + @Override + public PCollection> expand( + PCollection> input) { + return input + .apply(ParDo.of(new MapStringToDlpRow(getColumnDelimiter()))) + .apply("Batch Contents", ParDo.of(new BatchRequestForDLP(getBatchSizeBytes()))) + .apply( + "DLPDeidentify", + ParDo.of( + new DeidentifyText( + getProjectId(), + getInspectTemplateName(), + getDeidentifyTemplateName(), + getInspectConfig(), + getDeidentifyConfig(), + getHeaderColumns()))); + } + + /** DoFn performing calls to Cloud DLP service on GCP. */ + static class DeidentifyText + extends DoFn>, KV> { + private final String projectId; + private final String inspectTemplateName; + private final String deidentifyTemplateName; + private final InspectConfig inspectConfig; + private final DeidentifyConfig deidentifyConfig; + private final PCollectionView> headerColumns; + private transient DeidentifyContentRequest.Builder requestBuilder; + private transient DlpServiceClient dlpServiceClient; + + @Setup + public void setup() throws IOException { + requestBuilder = + DeidentifyContentRequest.newBuilder().setParent(ProjectName.of(projectId).toString()); + if (inspectTemplateName != null) { + requestBuilder.setInspectTemplateName(inspectTemplateName); + } + if (inspectConfig != null) { + requestBuilder.setInspectConfig(inspectConfig); + } + if (deidentifyConfig != null) { + requestBuilder.setDeidentifyConfig(deidentifyConfig); + } + if (deidentifyTemplateName != null) { + requestBuilder.setDeidentifyTemplateName(deidentifyTemplateName); + } + dlpServiceClient = DlpServiceClient.create(); + } + + @Teardown + public void teardown() { + dlpServiceClient.close(); + } + + /** + * @param projectId ID of GCP project that should be used for deidentification. + * @param inspectTemplateName Template name for inspection. Optional. + * @param deidentifyTemplateName Template name for deidentification. Either this or + * deidentifyConfig is required. + * @param inspectConfig Configuration object for inspection. Optional. + * @param deidentifyConfig Deidentification config containing data transformations. Either this + * or deidentifyTemplateName is required. + * @param headerColumns Header row of the table if applicable. + */ + public DeidentifyText( + String projectId, + String inspectTemplateName, + String deidentifyTemplateName, + InspectConfig inspectConfig, + DeidentifyConfig deidentifyConfig, + PCollectionView> headerColumns) { + this.projectId = projectId; + this.inspectTemplateName = inspectTemplateName; + this.deidentifyTemplateName = deidentifyTemplateName; + this.inspectConfig = inspectConfig; + this.deidentifyConfig = deidentifyConfig; + this.headerColumns = headerColumns; + } + + @ProcessElement + public void processElement(ProcessContext c) throws IOException { + String fileName = c.element().getKey(); + List dlpTableHeaders; + if (headerColumns != null) { + dlpTableHeaders = + c.sideInput(headerColumns).stream() + .map(header -> FieldId.newBuilder().setName(header).build()) + .collect(Collectors.toList()); + } else { + // handle unstructured input + dlpTableHeaders = new ArrayList<>(); + dlpTableHeaders.add(FieldId.newBuilder().setName("value").build()); + } + Table table = + Table.newBuilder() + .addAllHeaders(dlpTableHeaders) + .addAllRows(c.element().getValue()) + .build(); + ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); + this.requestBuilder.setItem(contentItem); + DeidentifyContentResponse response = + dlpServiceClient.deidentifyContent(this.requestBuilder.build()); + c.output(KV.of(fileName, response)); + } + } +} diff --git a/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPInspectText.java b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPInspectText.java new file mode 100644 index 000000000000..ff58674fbfcc --- /dev/null +++ b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPInspectText.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.auto.value.AutoValue; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.ProjectName; +import com.google.privacy.dlp.v2.Table; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.sdk.annotations.Experimental; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionView; + +/** + * A {@link PTransform} connecting to Cloud DLP (https://cloud.google.com/dlp/docs/libraries) and + * inspecting text for identifying data according to provided settings. The transform supports both + * delimited columnar input data (eg. CSV) and unstructured input. + * + *

If the headerColumns property is set and a sideinput with table headers is added to the + * PTransform, delimiter also should be set, else the results will be incorrect. If headerColumns is + * neither set nor passed as sideinput, input is assumed to be unstructured. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + * + *

The transform consumes {@link KV} of {@link String}s (assumed to be filename as key and + * contents as value) and outputs {@link KV} of {@link String} (eg. filename) and {@link + * InspectContentResponse}, which will contain a list of {@link + * com.google.privacy.dlp.v2.InspectResult} for the user to consume. + * + *

Either inspectTemplateName (String) or inspectConfig {@link InspectConfig} need to be set. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + */ +@Experimental +@AutoValue +public abstract class DLPInspectText + extends PTransform< + PCollection>, PCollection>> { + + public static final Integer DLP_PAYLOAD_LIMIT_BYTES = 524000; + + /** @return Template name for data inspection. */ + @Nullable + public abstract String getInspectTemplateName(); + + /** + * @return Configuration object for data inspection. If present, supersedes the template settings. + */ + @Nullable + public abstract InspectConfig getInspectConfig(); + + /** @return Size of input elements batch to be sent to Cloud DLP service in one request. */ + public abstract Integer getBatchSizeBytes(); + + /** @return ID of Google Cloud project to be used when deidentifying data. */ + public abstract String getProjectId(); + + /** @return Delimiter to be used when splitting values from input strings into columns. */ + @Nullable + public abstract String getColumnDelimiter(); + + /** @return List of column names if the input KV value is a delimited row. */ + @Nullable + public abstract PCollectionView> getHeaderColumns(); + + @AutoValue.Builder + public abstract static class Builder { + /** @param inspectTemplateName Template name for data inspection. */ + public abstract Builder setInspectTemplateName(String inspectTemplateName); + + /** + * @param inspectConfig Configuration object for data inspection. If present, supersedes the + * template settings. + */ + public abstract Builder setInspectConfig(InspectConfig inspectConfig); + + /** + * @param batchSize Size of input elements batch to be sent to Cloud DLP service in one request. + */ + public abstract Builder setBatchSizeBytes(Integer batchSize); + + /** @param projectId ID of Google Cloud project to be used when deidentifying data. */ + public abstract Builder setProjectId(String projectId); + + /** + * @param delimiter Delimiter to be used when splitting values from input strings into columns. + */ + public abstract Builder setColumnDelimiter(String delimiter); + + /** @param headerColumns List of column names if the input KV value is a delimited row. */ + public abstract Builder setHeaderColumns(PCollectionView> headerColumns); + + abstract DLPInspectText autoBuild(); + + public DLPInspectText build() { + DLPInspectText inspectText = autoBuild(); + if (inspectText.getInspectTemplateName() == null && inspectText.getInspectConfig() == null) { + throw new IllegalArgumentException( + "Either inspectTemplateName or inspectConfig must be supplied!"); + } + if (inspectText.getBatchSizeBytes() > DLP_PAYLOAD_LIMIT_BYTES) { + throw new IllegalArgumentException( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLP_PAYLOAD_LIMIT_BYTES)); + } + if (inspectText.getColumnDelimiter() == null && inspectText.getHeaderColumns() != null) { + throw new IllegalArgumentException( + "Column delimiter should be set if headers are present."); + } + if (inspectText.getHeaderColumns() == null && inspectText.getColumnDelimiter() != null) { + throw new IllegalArgumentException( + "Column headers should be supplied when delimiter is present."); + } + return inspectText; + } + } + + public static Builder newBuilder() { + return new AutoValue_DLPInspectText.Builder(); + } + + /** + * The transform converts the contents of input PCollection into {@link Table.Row}s and then calls + * Cloud DLP service to perform the data inspection according to provided settings. + * + * @param input input PCollection + * @return PCollection after transformations + */ + @Override + public PCollection> expand( + PCollection> input) { + return input + .apply(ParDo.of(new MapStringToDlpRow(getColumnDelimiter()))) + .apply("Batch Contents", ParDo.of(new BatchRequestForDLP(getBatchSizeBytes()))) + .apply( + "DLPInspect", + ParDo.of( + new InspectData( + getProjectId(), + getInspectTemplateName(), + getInspectConfig(), + getHeaderColumns()))); + } + + /** Performs calls to Cloud DLP service on GCP to inspect input data. */ + static class InspectData + extends DoFn>, KV> { + private final String projectId; + private final String inspectTemplateName; + private final InspectConfig inspectConfig; + private final PCollectionView> headerColumns; + private transient DlpServiceClient dlpServiceClient; + private transient InspectContentRequest.Builder requestBuilder; + + /** + * @param projectId ID of GCP project that should be used for data inspection. + * @param inspectTemplateName Template name for inspection. + * @param inspectConfig Configuration object for inspection. + * @param headerColumns Header row of the table if applicable. + */ + public InspectData( + String projectId, + String inspectTemplateName, + InspectConfig inspectConfig, + PCollectionView> headerColumns) { + this.projectId = projectId; + this.inspectTemplateName = inspectTemplateName; + this.inspectConfig = inspectConfig; + this.headerColumns = headerColumns; + } + + @Setup + public void setup() throws IOException { + this.requestBuilder = + InspectContentRequest.newBuilder().setParent(ProjectName.of(this.projectId).toString()); + if (inspectTemplateName != null) { + requestBuilder.setInspectTemplateName(this.inspectTemplateName); + } + if (inspectConfig != null) { + requestBuilder.setInspectConfig(inspectConfig); + } + dlpServiceClient = DlpServiceClient.create(); + } + + @Teardown + public void teardown() { + dlpServiceClient.close(); + } + + @ProcessElement + public void processElement(ProcessContext c) throws IOException { + List tableHeaders; + if (headerColumns != null) { + tableHeaders = + c.sideInput(headerColumns).stream() + .map(header -> FieldId.newBuilder().setName(header).build()) + .collect(Collectors.toList()); + } else { + tableHeaders = new ArrayList<>(); + tableHeaders.add(FieldId.newBuilder().setName("value").build()); + } + Table table = + Table.newBuilder().addAllHeaders(tableHeaders).addAllRows(c.element().getValue()).build(); + ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); + this.requestBuilder.setItem(contentItem); + InspectContentResponse response = + dlpServiceClient.inspectContent(this.requestBuilder.build()); + c.output(KV.of(c.element().getKey(), response)); + } + } +} diff --git a/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyText.java b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyText.java new file mode 100644 index 000000000000..5dc20cbc3697 --- /dev/null +++ b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyText.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.auto.value.AutoValue; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.ProjectName; +import com.google.privacy.dlp.v2.ReidentifyContentRequest; +import com.google.privacy.dlp.v2.ReidentifyContentResponse; +import com.google.privacy.dlp.v2.Table; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.sdk.annotations.Experimental; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionView; + +/** + * A {@link PTransform} connecting to Cloud DLP (https://cloud.google.com/dlp/docs/libraries) and + * inspecting text for identifying data according to provided settings. + * + *

The transform supports both delimited columnar input data and unstructured input. + * + *

If the headerColumns property is set and a sideinput with headers is added to the PTransform, + * delimiter also should be set, else the results will be incorrect. If headerColumns is neither set + * nor passed as sideinput, input is assumed to be unstructured. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + * + *

The transform consumes {@link KV} of {@link String}s (assumed to be filename as key and + * contents as value) and outputs {@link KV} of {@link String} (eg. filename) and {@link + * ReidentifyContentResponse}, which will contain {@link Table} of results for the user to consume. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + * + *

Either reidentifyTemplateName {@link String} or reidentifyConfig {@link DeidentifyConfig} need + * to be set. inspectConfig {@link InspectConfig} and inspectTemplateName {@link String} are + * optional. + * + *

Batch size defines how big are batches sent to DLP at once in bytes. + */ +@Experimental +@AutoValue +public abstract class DLPReidentifyText + extends PTransform< + PCollection>, PCollection>> { + + public static final Integer DLP_PAYLOAD_LIMIT_BYTES = 524000; + + /** @return Template name for data inspection. */ + @Nullable + public abstract String getInspectTemplateName(); + + /** @return Template name for data reidentification. */ + @Nullable + public abstract String getReidentifyTemplateName(); + + /** + * @return Configuration object for data inspection. If present, supersedes the template settings. + */ + @Nullable + public abstract InspectConfig getInspectConfig(); + + /** @return Configuration object for reidentification. If present, supersedes the template. */ + @Nullable + public abstract DeidentifyConfig getReidentifyConfig(); + + /** @return Delimiter to be used when splitting values from input strings into columns. */ + @Nullable + public abstract String getColumnDelimiter(); + + /** @return List of column names if the input KV value is a delimited row. */ + @Nullable + public abstract PCollectionView> getHeaderColumns(); + + /** @return Size of input elements batch to be sent to Cloud DLP service in one request. */ + public abstract Integer getBatchSizeBytes(); + + /** @return ID of Google Cloud project to be used when deidentifying data. */ + public abstract String getProjectId(); + + @AutoValue.Builder + public abstract static class Builder { + /** @param inspectTemplateName Template name for data inspection. */ + public abstract Builder setInspectTemplateName(String inspectTemplateName); + + /** + * @param inspectConfig Configuration object for data inspection. If present, supersedes the + * template settings. + */ + public abstract Builder setInspectConfig(InspectConfig inspectConfig); + + /** + * @param reidentifyConfig Configuration object for data deidentification. If present, + * supersedes the template settings. + */ + public abstract Builder setReidentifyConfig(DeidentifyConfig reidentifyConfig); + + /** @param reidentifyTemplateName Template name for data deidentification. */ + public abstract Builder setReidentifyTemplateName(String reidentifyTemplateName); + + /** + * @param batchSize Size of input elements batch to be sent to Cloud DLP service in one request. + */ + public abstract Builder setBatchSizeBytes(Integer batchSize); + /** @param headerColumns List of column names if the input KV value is a delimited row. */ + public abstract Builder setHeaderColumns(PCollectionView> headerColumns); + + /** + * @param delimiter Delimiter to be used when splitting values from input strings into columns. + */ + public abstract Builder setColumnDelimiter(String delimiter); + + /** @param projectId ID of Google Cloud project to be used when deidentifying data. */ + public abstract Builder setProjectId(String projectId); + + abstract DLPReidentifyText autoBuild(); + + public DLPReidentifyText build() { + DLPReidentifyText dlpReidentifyText = autoBuild(); + if (dlpReidentifyText.getReidentifyConfig() == null + && dlpReidentifyText.getReidentifyTemplateName() == null) { + throw new IllegalArgumentException( + "Either reidentifyConfig or reidentifyTemplateName need to be set!"); + } + if (dlpReidentifyText.getBatchSizeBytes() > DLP_PAYLOAD_LIMIT_BYTES) { + throw new IllegalArgumentException( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLP_PAYLOAD_LIMIT_BYTES)); + } + if (dlpReidentifyText.getColumnDelimiter() == null + && dlpReidentifyText.getHeaderColumns() != null) { + throw new IllegalArgumentException( + "Column delimiter should be set if headers are present."); + } + if (dlpReidentifyText.getHeaderColumns() == null + && dlpReidentifyText.getColumnDelimiter() != null) { + throw new IllegalArgumentException( + "Column headers should be supplied when delimiter is present."); + } + return dlpReidentifyText; + } + } + + public static DLPReidentifyText.Builder newBuilder() { + return new AutoValue_DLPReidentifyText.Builder(); + } + + /** + * The transform converts the contents of input PCollection into {@link Table.Row}s and then calls + * Cloud DLP service to perform the reidentification according to provided settings. + * + * @param input input PCollection + * @return PCollection after transformations + */ + @Override + public PCollection> expand( + PCollection> input) { + return input + .apply(ParDo.of(new MapStringToDlpRow(getColumnDelimiter()))) + .apply("Batch Contents", ParDo.of(new BatchRequestForDLP(getBatchSizeBytes()))) + .apply( + "DLPReidentify", + ParDo.of( + new ReidentifyText( + getProjectId(), + getInspectTemplateName(), + getReidentifyTemplateName(), + getInspectConfig(), + getReidentifyConfig(), + getHeaderColumns()))); + } + + /** Performs the calls to Cloud DLP service on GCP. */ + static class ReidentifyText + extends DoFn>, KV> { + private final String projectId; + private final String inspectTemplateName; + private final String reidentifyTemplateName; + private final InspectConfig inspectConfig; + private final DeidentifyConfig reidentifyConfig; + private final PCollectionView> headerColumns; + private transient ReidentifyContentRequest.Builder requestBuilder; + private transient DlpServiceClient dlpServiceClient; + + @Setup + public void setup() throws IOException { + requestBuilder = + ReidentifyContentRequest.newBuilder().setParent(ProjectName.of(projectId).toString()); + if (inspectTemplateName != null) { + requestBuilder.setInspectTemplateName(inspectTemplateName); + } + if (inspectConfig != null) { + requestBuilder.setInspectConfig(inspectConfig); + } + if (reidentifyConfig != null) { + requestBuilder.setReidentifyConfig(reidentifyConfig); + } + if (reidentifyTemplateName != null) { + requestBuilder.setReidentifyTemplateName(reidentifyTemplateName); + } + dlpServiceClient = DlpServiceClient.create(); + } + + @Teardown + public void teardown() { + dlpServiceClient.close(); + } + + /** + * @param projectId ID of GCP project that should be used for deidentification. + * @param inspectTemplateName Template name for inspection. Optional. + * @param reidentifyTemplateName Template name for reidentification. Either this or + * reidentifyConfig is required. + * @param inspectConfig Configuration object for inspection. Optional. + * @param reidentifyConfig Reidentification config containing data transformations. Either this + * or reidentifyTemplateName is required. + * @param headerColumns Header row of the table if applicable. + */ + public ReidentifyText( + String projectId, + String inspectTemplateName, + String reidentifyTemplateName, + InspectConfig inspectConfig, + DeidentifyConfig reidentifyConfig, + PCollectionView> headerColumns) { + this.projectId = projectId; + this.inspectTemplateName = inspectTemplateName; + this.reidentifyTemplateName = reidentifyTemplateName; + this.inspectConfig = inspectConfig; + this.reidentifyConfig = reidentifyConfig; + this.headerColumns = headerColumns; + } + + @ProcessElement + public void processElement(ProcessContext context) throws IOException { + List tableHeaders; + if (headerColumns != null) { + tableHeaders = + context.sideInput(headerColumns).stream() + .map(header -> FieldId.newBuilder().setName(header).build()) + .collect(Collectors.toList()); + } else { + // handle unstructured input. + tableHeaders = new ArrayList<>(); + tableHeaders.add(FieldId.newBuilder().setName("value").build()); + } + Table table = + Table.newBuilder() + .addAllHeaders(tableHeaders) + .addAllRows(context.element().getValue()) + .build(); + ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); + this.requestBuilder.setItem(contentItem); + ReidentifyContentResponse response = + dlpServiceClient.reidentifyContent(requestBuilder.build()); + context.output(KV.of(context.element().getKey(), response)); + } + } +} diff --git a/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRow.java b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRow.java new file mode 100644 index 000000000000..944656c59fb1 --- /dev/null +++ b/sdks/java/extensions/ml/src/main/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRow.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Value; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; + +/** + * Maps {@link KV}s of {@link String}s into KV<{@link String}, {@link Table.Row}> for further + * processing in the DLP transforms. + * + *

If a delimiter of values isn't provided, input is assumed to be unstructured and the input KV + * value is saved in a single column of output {@link Table.Row}. + */ +class MapStringToDlpRow extends DoFn, KV> { + private final String delimiter; + + /** + * @param delimiter Delimiter of values in the delimited value row that may be in the value of + * input KV. + */ + public MapStringToDlpRow(String delimiter) { + this.delimiter = delimiter; + } + + @ProcessElement + public void processElement(ProcessContext context) { + Table.Row.Builder rowBuilder = Table.Row.newBuilder(); + String line = Objects.requireNonNull(context.element().getValue()); + if (delimiter != null) { + List values = Arrays.asList(line.split(delimiter)); + values.forEach( + value -> rowBuilder.addValues(Value.newBuilder().setStringValue(value).build())); + } else { + rowBuilder.addValues(Value.newBuilder().setStringValue(line).build()); + } + context.output(KV.of(context.element().getKey(), rowBuilder.build())); + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDlpTest.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDlpTest.java new file mode 100644 index 000000000000..3e9a74178929 --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/BatchRequestForDlpTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import static org.junit.Assert.assertEquals; + +import com.google.privacy.dlp.v2.Table; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BatchRequestForDlpTest { + + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + @Test + public void batchesRequests() { + PCollection>> batchedRows = + testPipeline + .apply(Create.of(KV.of("key", "value1"), KV.of("key", "value2"))) + .apply(ParDo.of(new MapStringToDlpRow(null))) + .apply(ParDo.of(new BatchRequestForDLP(524000))); + PAssert.that(batchedRows).satisfies(new VerifyPCollectionSize()); + testPipeline.run().waitUntilFinish(); + } + + private static class VerifyPCollectionSize + implements SerializableFunction>>, Void> { + @Override + public Void apply(Iterable>> input) { + List>> itemList = new ArrayList<>(); + input.forEach(itemList::add); + assertEquals(1, itemList.size()); + return null; + } + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyTextTest.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyTextTest.java new file mode 100644 index 000000000000..c2e5741d81db --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPDeidentifyTextTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import static org.junit.Assert.assertThrows; + +import java.util.List; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.View; +import org.apache.beam.sdk.values.PCollectionView; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DLPDeidentifyTextTest { + + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + private static final Integer BATCH_SIZE_SMALL = 200; + private static final String DELIMITER = ";"; + private static final String TEMPLATE_NAME = "test_template"; + private static final String PROJECT_ID = "test_id"; + + @Test + public void throwsExceptionWhenDeidentifyConfigAndTemplatesAreEmpty() { + assertThrows( + "Either deidentifyConfig or deidentifyTemplateName need to be set!", + IllegalArgumentException.class, + () -> + DLPDeidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsNullAndHeadersAreSet() { + PCollectionView> header = + testPipeline.apply(Create.of("header")).apply(View.asList()); + assertThrows( + "Column delimiter should be set if headers are present.", + IllegalArgumentException.class, + () -> + DLPDeidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setDeidentifyTemplateName(TEMPLATE_NAME) + .setHeaderColumns(header) + .build()); + testPipeline.run().waitUntilFinish(); + } + + @Test + public void throwsExceptionWhenBatchSizeIsTooLarge() { + assertThrows( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLPDeidentifyText.DLP_PAYLOAD_LIMIT_BYTES), + IllegalArgumentException.class, + () -> + DLPDeidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(Integer.MAX_VALUE) + .setDeidentifyTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsSetAndHeadersAreNot() { + assertThrows( + "Column headers should be supplied when delimiter is present.", + IllegalArgumentException.class, + () -> + DLPDeidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setDeidentifyTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPInspectTextTest.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPInspectTextTest.java new file mode 100644 index 000000000000..46c8fa3a3096 --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPInspectTextTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import static org.junit.Assert.assertThrows; + +import java.util.List; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.View; +import org.apache.beam.sdk.values.PCollectionView; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DLPInspectTextTest { + + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + private static final Integer BATCH_SIZE_SMALL = 200; + private static final String DELIMITER = ";"; + private static final String TEMPLATE_NAME = "test_template"; + private static final String PROJECT_ID = "test_id"; + + @Test + public void throwsExceptionWhenDeidentifyConfigAndTemplatesAreEmpty() { + assertThrows( + "Either inspectTemplateName or inspectConfig must be supplied!", + IllegalArgumentException.class, + () -> + DLPInspectText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsNullAndHeadersAreSet() { + PCollectionView> header = + testPipeline.apply(Create.of("header")).apply(View.asList()); + assertThrows( + "Column delimiter should be set if headers are present.", + IllegalArgumentException.class, + () -> + DLPInspectText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setInspectTemplateName(TEMPLATE_NAME) + .setHeaderColumns(header) + .build()); + testPipeline.run().waitUntilFinish(); + } + + @Test + public void throwsExceptionWhenBatchSizeIsTooLarge() { + assertThrows( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLPInspectText.DLP_PAYLOAD_LIMIT_BYTES), + IllegalArgumentException.class, + () -> + DLPInspectText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(Integer.MAX_VALUE) + .setInspectTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsSetAndHeadersAreNot() { + assertThrows( + "Column headers should be supplied when delimiter is present.", + IllegalArgumentException.class, + () -> + DLPInspectText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setInspectTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyTextTest.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyTextTest.java new file mode 100644 index 000000000000..9fc0f0a9d7a7 --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPReidentifyTextTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import static org.junit.Assert.assertThrows; + +import java.util.List; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.View; +import org.apache.beam.sdk.values.PCollectionView; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DLPReidentifyTextTest { + + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + private static final Integer BATCH_SIZE_SMALL = 200; + private static final String DELIMITER = ";"; + private static final String TEMPLATE_NAME = "test_template"; + private static final String PROJECT_ID = "test_id"; + + @Test + public void throwsExceptionWhenDeidentifyConfigAndTemplatesAreEmpty() { + assertThrows( + "Either reidentifyConfig or reidentifyTemplateName need to be set!", + IllegalArgumentException.class, + () -> + DLPReidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsNullAndHeadersAreSet() { + PCollectionView> header = + testPipeline.apply(Create.of("header")).apply(View.asList()); + assertThrows( + "Column delimiter should be set if headers are present.", + IllegalArgumentException.class, + () -> + DLPReidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setReidentifyTemplateName(TEMPLATE_NAME) + .setHeaderColumns(header) + .build()); + testPipeline.run().waitUntilFinish(); + } + + @Test + public void throwsExceptionWhenBatchSizeIsTooLarge() { + assertThrows( + String.format( + "Batch size is too large! It should be smaller or equal than %d.", + DLPDeidentifyText.DLP_PAYLOAD_LIMIT_BYTES), + IllegalArgumentException.class, + () -> + DLPReidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(Integer.MAX_VALUE) + .setReidentifyTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } + + @Test + public void throwsExceptionWhenDelimiterIsSetAndHeadersAreNot() { + assertThrows( + "Column headers should be supplied when delimiter is present.", + IllegalArgumentException.class, + () -> + DLPReidentifyText.newBuilder() + .setProjectId(PROJECT_ID) + .setBatchSizeBytes(BATCH_SIZE_SMALL) + .setReidentifyTemplateName(TEMPLATE_NAME) + .setColumnDelimiter(DELIMITER) + .build()); + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPTextOperationsIT.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPTextOperationsIT.java new file mode 100644 index 000000000000..0ebbfb7b0008 --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/DLPTextOperationsIT.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import static org.junit.Assert.assertTrue; + +import com.google.privacy.dlp.v2.CharacterMaskConfig; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DLPTextOperationsIT { + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + private static final String IDENTIFYING_TEXT = "mary.sue@example.com"; + private static InfoType emailAddress = InfoType.newBuilder().setName("EMAIL_ADDRESS").build(); + private static final InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addInfoTypes(emailAddress) + .setMinLikelihood(Likelihood.LIKELY) + .build(); + + @Test + public void inspectsText() { + String projectId = testPipeline.getOptions().as(GcpOptions.class).getProject(); + PCollection> inspectionResult = + testPipeline + .apply(Create.of(KV.of("", IDENTIFYING_TEXT))) + .apply( + DLPInspectText.newBuilder() + .setBatchSizeBytes(524000) + .setProjectId(projectId) + .setInspectConfig(inspectConfig) + .build()); + PAssert.that(inspectionResult).satisfies(new VerifyInspectionResult()); + testPipeline.run().waitUntilFinish(); + } + + @Test + public void deidentifiesText() { + String projectId = testPipeline.getOptions().as(GcpOptions.class).getProject(); + + PCollection> deidentificationResult = + testPipeline + .apply(Create.of(KV.of("", IDENTIFYING_TEXT))) + .apply( + DLPDeidentifyText.newBuilder() + .setBatchSizeBytes(524000) + .setProjectId(projectId) + .setDeidentifyConfig(getDeidentifyConfig()) + .build()); + PAssert.that(deidentificationResult) + .satisfies(new VerifyDeidentificationResult("####################")); + testPipeline.run().waitUntilFinish(); + } + + private DeidentifyConfig getDeidentifyConfig() { + CharacterMaskConfig characterMaskConfig = + CharacterMaskConfig.newBuilder().setMaskingCharacter("#").build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder().setCharacterMaskConfig(characterMaskConfig).build(); + InfoTypeTransformations.InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformations.InfoTypeTransformation.newBuilder() + .addInfoTypes(emailAddress) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + return DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build()) + .build(); + } + + private static class VerifyInspectionResult + implements SerializableFunction>, Void> { + @Override + public Void apply(Iterable> input) { + List matches = new ArrayList<>(); + input.forEach( + item -> { + List resultList = item.getValue().getResult().getFindingsList(); + matches.add( + resultList.stream() + .anyMatch(finding -> finding.getInfoType().equals(emailAddress))); + }); + assertTrue(matches.contains(Boolean.TRUE)); + return null; + } + } + + private static class VerifyDeidentificationResult + implements SerializableFunction>, Void> { + private final String expectedValue; + + public VerifyDeidentificationResult(String expectedValue) { + this.expectedValue = expectedValue; + } + + @Override + public Void apply(Iterable> input) { + List matches = new ArrayList<>(); + input.forEach( + item -> { + item.getValue() + .getItem() + .getTable() + .getRowsList() + .forEach( + row -> + matches.add( + row.getValuesList().stream() + .anyMatch(value -> value.getStringValue().equals(expectedValue)))); + assertTrue( + item.getValue() + .getItem() + .getTable() + .getHeadersList() + .contains(FieldId.newBuilder().setName("value").build())); + }); + assertTrue(matches.contains(Boolean.TRUE)); + return null; + } + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRowTest.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRowTest.java new file mode 100644 index 000000000000..577a5dc7ed34 --- /dev/null +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/MapStringToDlpRowTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.ml; + +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Value; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class MapStringToDlpRowTest { + @Rule public TestPipeline testPipeline = TestPipeline.create(); + + @Test + public void mapsStringToRow() { + PCollection> rowCollection = + testPipeline + .apply(Create.of(KV.of("key", "value"))) + .apply(ParDo.of(new MapStringToDlpRow(null))); + PAssert.that(rowCollection) + .containsInAnyOrder( + KV.of( + "key", + Table.Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("value").build()) + .build())); + testPipeline.run().waitUntilFinish(); + } + + @Test + public void mapsDelimitedStringToRow() { + PCollection> rowCollection = + testPipeline + .apply(Create.of(KV.of("key", "value,secondValue"))) + .apply(ParDo.of(new MapStringToDlpRow(","))); + PAssert.that(rowCollection) + .containsInAnyOrder( + KV.of( + "key", + Table.Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("value").build()) + .addValues(Value.newBuilder().setStringValue("secondValue").build()) + .build())); + testPipeline.run().waitUntilFinish(); + } +} diff --git a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/VideoIntelligenceIT.java b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/VideoIntelligenceIT.java index 642722562384..20e9145d5e78 100644 --- a/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/VideoIntelligenceIT.java +++ b/sdks/java/extensions/ml/src/test/java/org/apache/beam/sdk/extensions/ml/VideoIntelligenceIT.java @@ -58,7 +58,7 @@ private static class VerifyVideoAnnotationResult @Override public Void apply(Iterable> input) { List labelEvaluations = new ArrayList<>(); - input.forEach(findStringMatchesInVideoAnnotationResultList(labelEvaluations, "dinosaur")); + input.forEach(findStringMatchesInVideoAnnotationResultList(labelEvaluations, "bicycle")); assertEquals(Boolean.TRUE, labelEvaluations.contains(Boolean.TRUE)); return null; } @@ -73,9 +73,9 @@ private Consumer> findStringMatchesInVideoAnnotatio private boolean entityWithDescriptionFoundInSegmentLabels( String toMatch, VideoAnnotationResults result) { - return result.getSegmentLabelAnnotationsList().stream() + return result.getSegmentPresenceLabelAnnotationsList().stream() .anyMatch( - labelAnnotation -> labelAnnotation.getEntity().getDescription().equals(toMatch)); + labelAnnotation -> labelAnnotation.getEntity().getDescription().contains(toMatch)); } } } diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index 19a32709522b..998d3cae5f2d 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -30,6 +30,7 @@ ext.summary = """Contains code to run a SQL Expansion Service.""" dependencies { compile project(path: ":sdks:java:extensions:sql") + compile project(path: ":sdks:java:extensions:sql:zetasql") compile project(path: ":sdks:java:expansion-service") } diff --git a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java new file mode 100644 index 000000000000..cec0a867bc0d --- /dev/null +++ b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql.expansion; + +import com.google.auto.service.AutoService; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.sdk.expansion.ExternalTransformRegistrar; +import org.apache.beam.sdk.extensions.sql.SqlTransform; +import org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner; +import org.apache.beam.sdk.extensions.sql.impl.QueryPlanner; +import org.apache.beam.sdk.extensions.sql.zetasql.ZetaSQLQueryPlanner; +import org.apache.beam.sdk.transforms.ExternalTransformBuilder; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; + +@AutoService(ExternalTransformRegistrar.class) +public class ExternalSqlTransformRegistrar implements ExternalTransformRegistrar { + private static final String URN = "beam:external:java:sql:v1"; + private static final ImmutableMap> DIALECTS = + ImmutableMap.>builder() + .put("zetasql", ZetaSQLQueryPlanner.class) + .put("calcite", CalciteQueryPlanner.class) + .build(); + + @Override + public Map> knownBuilders() { + return org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap.of( + URN, Builder.class); + } + + public static class Configuration { + String query; + String dialect; + + public void setQuery(String query) { + this.query = query; + } + + public void setDialect(@Nullable String dialect) { + this.dialect = dialect; + } + } + + private static class Builder + implements ExternalTransformBuilder> { + @Override + public PTransform> buildExternal(Configuration configuration) { + SqlTransform transform = SqlTransform.query(configuration.query); + if (configuration.dialect != null) { + Class queryPlanner = + DIALECTS.get(configuration.dialect.toLowerCase()); + if (queryPlanner == null) { + throw new IllegalArgumentException( + String.format( + "Received unknown SQL Dialect '%s'. Known dialects: %s", + configuration.dialect, DIALECTS.keySet())); + } + transform = transform.withQueryPlannerClass(queryPlanner); + } + return transform; + } + } +} diff --git a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/package-info.java b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/package-info.java new file mode 100644 index 000000000000..1dcf3fca9e16 --- /dev/null +++ b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** External Transform Registration for Beam SQL. */ +package org.apache.beam.sdk.extensions.sql.expansion; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java index 496dd6d37f46..12ce337f9685 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java @@ -17,7 +17,6 @@ */ package org.apache.beam.sdk.extensions.sql; -import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; import java.util.Collections; import java.util.HashMap; @@ -25,7 +24,6 @@ import java.util.Map; import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; -import org.apache.beam.sdk.expansion.ExternalTransformRegistrar; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv.BeamSqlEnvBuilder; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlPipelineOptions; @@ -37,7 +35,6 @@ import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider; import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; import org.apache.beam.sdk.transforms.Combine; -import org.apache.beam.sdk.transforms.ExternalTransformBuilder; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; @@ -273,8 +270,7 @@ static Builder builder() { } @AutoValue.Builder - abstract static class Builder - implements ExternalTransformBuilder> { + abstract static class Builder { abstract Builder setQueryString(String queryString); abstract Builder setQueryParameters(QueryParameters queryParameters); @@ -292,19 +288,6 @@ abstract static class Builder abstract Builder setQueryPlannerClassName(@Nullable String queryPlannerClassName); abstract SqlTransform build(); - - @Override - public PTransform> buildExternal( - External.Configuration configuration) { - return builder() - .setQueryString(configuration.query) - .setQueryParameters(QueryParameters.ofNone()) - .setUdafDefinitions(Collections.emptyList()) - .setUdfDefinitions(Collections.emptyList()) - .setTableProviderMap(Collections.emptyMap()) - .setAutoUdfUdafLoad(false) - .build(); - } } @AutoValue @@ -330,24 +313,4 @@ static UdafDefinition of(String udafName, Combine.CombineFn combineFn) { return new AutoValue_SqlTransform_UdafDefinition(udafName, combineFn); } } - - @AutoService(ExternalTransformRegistrar.class) - public static class External implements ExternalTransformRegistrar { - - private static final String URN = "beam:external:java:sql:v1"; - - @Override - public Map> knownBuilders() { - return org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap.of( - URN, AutoValue_SqlTransform.Builder.class); - } - - public static class Configuration { - String query; - - public void setQuery(String query) { - this.query = query; - } - } - } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java index 8496a712fef0..9742a3384b33 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java @@ -45,6 +45,7 @@ * Beam documentation on how to run pipelines. */ class BeamSqlExample { + public static void main(String[] args) { PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); Pipeline p = Pipeline.create(options); @@ -66,19 +67,21 @@ public static void main(String[] args) { inputTable.apply(SqlTransform.query("select c1, c2, c3 from PCOLLECTION where c1 > 1")); // print the output record of case 1; - outputStream.apply( - "log_result", - MapElements.via( - new SimpleFunction() { - @Override - public Row apply(Row input) { - // expect output: - // PCOLLECTION: [3, row, 3.0] - // PCOLLECTION: [2, row, 2.0] - System.out.println("PCOLLECTION: " + input.getValues()); - return input; - } - })); + outputStream + .apply( + "log_result", + MapElements.via( + new SimpleFunction() { + @Override + public Row apply(Row input) { + // expect output: + // PCOLLECTION: [3, row, 3.0] + // PCOLLECTION: [2, row, 2.0] + System.out.println("PCOLLECTION: " + input.getValues()); + return input; + } + })) + .setRowSchema(type); // Case 2. run the query with SqlTransform.query over result PCollection of case 1. PCollection outputStream2 = @@ -86,18 +89,21 @@ public Row apply(Row input) { .apply(SqlTransform.query("select c2, sum(c3) from CASE1_RESULT group by c2")); // print the output record of case 2; - outputStream2.apply( - "log_result", - MapElements.via( - new SimpleFunction() { - @Override - public Row apply(Row input) { - // expect output: - // CASE1_RESULT: [row, 5.0] - System.out.println("CASE1_RESULT: " + input.getValues()); - return input; - } - })); + outputStream2 + .apply( + "log_result", + MapElements.via( + new SimpleFunction() { + @Override + public Row apply(Row input) { + // expect output: + // CASE1_RESULT: [row, 5.0] + System.out.println("CASE1_RESULT: " + input.getValues()); + return input; + } + })) + .setRowSchema( + Schema.builder().addStringField("stringField").addDoubleField("doubleField").build()); p.run().waitUntilFinish(); } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java index ab76a7b897a9..bdc5f4c1bffa 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java @@ -20,15 +20,11 @@ import static org.apache.beam.vendor.calcite.v1_20_0.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; -import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSetOperatorsTransforms; -import org.apache.beam.sdk.schemas.transforms.CoGroup; -import org.apache.beam.sdk.schemas.transforms.CoGroup.By; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Sets; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; /** @@ -75,20 +71,27 @@ public PCollection expand(PCollectionList inputs) { + rightWindow); } - // TODO: We may want to preaggregate the counts first using Group instead of calling CoGroup and - // measuring the - // iterable size. If on average there are duplicates in the input, this will be faster. - final String lhsTag = "lhs"; - final String rhsTag = "rhs"; - PCollection joined = - PCollectionTuple.of(lhsTag, leftRows, rhsTag, rightRows) - .apply("CoGroup", CoGroup.join(By.fieldNames("*"))); - return joined - .apply( - "FilterResults", - ParDo.of( - new BeamSetOperatorsTransforms.SetOperatorFilteringDoFn( - lhsTag, rhsTag, opType, all))) - .setRowSchema(joined.getSchema().getField("key").getType().getRowSchema()); + switch (opType) { + case UNION: + if (all) { + return leftRows.apply(Sets.unionAll(rightRows)); + } else { + return leftRows.apply(Sets.unionDistinct(rightRows)); + } + case INTERSECT: + if (all) { + return leftRows.apply(Sets.intersectAll(rightRows)); + } else { + return leftRows.apply(Sets.intersectDistinct(rightRows)); + } + case MINUS: + if (all) { + return leftRows.apply(Sets.exceptAll(rightRows)); + } else { + return leftRows.apply(Sets.exceptDistinct(rightRows)); + } + default: + throw new IllegalStateException("Unexpected set operation value: " + opType); + } } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java index 74d6ccae41e4..6381fd96b787 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java @@ -29,8 +29,11 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.WithTimestamps; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.Row; @@ -99,14 +102,32 @@ public PCollection expand(PCollectionList input) { RexInputRef wmCol = (RexInputRef) call.getOperands().get(1); PCollection upstream = input.get(0); Schema outputSchema = CalciteUtils.toSchema(getRowType()); - return upstream - .apply( - ParDo.of( - new FixedWindowDoFn( - FixedWindows.of(durationParameter(call.getOperands().get(2))), - wmCol.getIndex(), - outputSchema))) - .setRowSchema(outputSchema); + FixedWindows windowFn = FixedWindows.of(durationParameter(call.getOperands().get(2))); + PCollection streamWithWindowMetadata = + upstream + .apply(ParDo.of(new FixedWindowDoFn(windowFn, wmCol.getIndex(), outputSchema))) + .setRowSchema(outputSchema); + + PCollection windowedStream = + assignTimestampsAndWindow( + streamWithWindowMetadata, wmCol.getIndex(), (WindowFn) windowFn); + + return windowedStream; + } + + /** Extract timestamps from the windowFieldIndex, then window into windowFns. */ + private PCollection assignTimestampsAndWindow( + PCollection upstream, int windowFieldIndex, WindowFn windowFn) { + PCollection windowedStream; + windowedStream = + upstream + .apply( + "assignEventTimestamp", + WithTimestamps.of(row -> row.getDateTime(windowFieldIndex).toInstant()) + .withAllowedTimestampSkew(new Duration(Long.MAX_VALUE))) + .setCoder(upstream.getCoder()) + .apply(Window.into(windowFn)); + return windowedStream; } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/udf/BuiltinHashFunctions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/udf/BuiltinHashFunctions.java new file mode 100644 index 000000000000..c3fc82b1fdb5 --- /dev/null +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/udf/BuiltinHashFunctions.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql.impl.udf; + +import com.google.auto.service.AutoService; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.linq4j.function.Strict; + +/** Hash Functions. */ +@AutoService(BeamBuiltinFunctionProvider.class) +public class BuiltinHashFunctions extends BeamBuiltinFunctionProvider { + + /** + * MD5(X) + * + *

Calculates the MD5 digest and returns the value as a 16 element {@code byte[]}. + */ + @UDF( + funcName = "MD5", + parameterArray = {Schema.TypeName.STRING}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] md5String(String str) { + return org.apache.commons.codec.digest.DigestUtils.md5(str); + } + + /** + * MD5(X) + * + *

Calculates the MD5 digest and returns the value as a 16 element {@code byte[]}. + */ + @UDF( + funcName = "MD5", + parameterArray = {Schema.TypeName.BYTES}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] md5Bytes(byte[] bytes) { + return org.apache.commons.codec.digest.DigestUtils.md5(bytes); + } + + /** + * SHA1(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA1", + parameterArray = {Schema.TypeName.STRING}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha1String(String str) { + return org.apache.commons.codec.digest.DigestUtils.sha1(str); + } + + /** + * SHA1(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA1", + parameterArray = {Schema.TypeName.BYTES}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha1Bytes(byte[] bytes) { + return org.apache.commons.codec.digest.DigestUtils.sha1(bytes); + } + + /** + * SHA256(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA256", + parameterArray = {Schema.TypeName.STRING}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha256String(String str) { + return org.apache.commons.codec.digest.DigestUtils.sha256(str); + } + + /** + * SHA256(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA256", + parameterArray = {Schema.TypeName.BYTES}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha256Bytes(byte[] bytes) { + return org.apache.commons.codec.digest.DigestUtils.sha256(bytes); + } + + /** + * SHA512(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA512", + parameterArray = {Schema.TypeName.STRING}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha512String(String str) { + return org.apache.commons.codec.digest.DigestUtils.sha512(str); + } + + /** + * SHA512(X) + * + *

Calculates the SHA-1 digest and returns the value as a {@code byte[]}. + */ + @UDF( + funcName = "SHA512", + parameterArray = {Schema.TypeName.BYTES}, + returnType = Schema.TypeName.BYTES) + @Strict + public byte[] sha512Bytes(byte[] bytes) { + return org.apache.commons.codec.digest.DigestUtils.sha512(bytes); + } +} diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java index ad26d4a27f5b..4298c0711cc4 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java @@ -213,6 +213,12 @@ public static void prepareClass() throws ParseException { "TO_HEX", "abcABC".getBytes(UTF_8), "TO_HEX", + "abcABCжщфЖЩФ".getBytes(UTF_8), + "HashingFn", + "foobar".getBytes(UTF_8), + "HashingFn", + " ".getBytes(UTF_8), + "HashingFn", "abcABCжщфЖЩФ".getBytes(UTF_8)) .getRows(); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSalUhfSpecialTypeAndValueTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSalUhfSpecialTypeAndValueTest.java index 1370c62db0f9..db87eb77a203 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSalUhfSpecialTypeAndValueTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSalUhfSpecialTypeAndValueTest.java @@ -27,6 +27,7 @@ import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; +import org.apache.commons.codec.digest.DigestUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -167,4 +168,72 @@ public void testRightPad() throws Exception { resultRow8); pipeline.run().waitUntilFinish(); } + + @Test + public void testMd5() throws Exception { + Schema resultType = Schema.builder().addByteArrayField("field").build(); + Row resultRow1 = + Row.withSchema(resultType).addValues(DigestUtils.md5("foobar".getBytes(UTF_8))).build(); + Row resultRow2 = + Row.withSchema(resultType).addValues(DigestUtils.md5(" ".getBytes(UTF_8))).build(); + Row resultRow3 = + Row.withSchema(resultType) + .addValues(DigestUtils.md5("abcABCжщфЖЩФ".getBytes(UTF_8))) + .build(); + String sql = "SELECT MD5(f_bytes) FROM PCOLLECTION WHERE f_func = 'HashingFn'"; + PCollection result = boundedInputBytes.apply("testUdf", SqlTransform.query(sql)); + PAssert.that(result).containsInAnyOrder(resultRow1, resultRow2, resultRow3); + pipeline.run().waitUntilFinish(); + } + + @Test + public void testSHA1() throws Exception { + Schema resultType = Schema.builder().addByteArrayField("field").build(); + Row resultRow1 = + Row.withSchema(resultType).addValues(DigestUtils.sha1("foobar".getBytes(UTF_8))).build(); + Row resultRow2 = + Row.withSchema(resultType).addValues(DigestUtils.sha1(" ".getBytes(UTF_8))).build(); + Row resultRow3 = + Row.withSchema(resultType) + .addValues(DigestUtils.sha1("abcABCжщфЖЩФ".getBytes(UTF_8))) + .build(); + String sql = "SELECT SHA1(f_bytes) FROM PCOLLECTION WHERE f_func = 'HashingFn'"; + PCollection result = boundedInputBytes.apply("testUdf", SqlTransform.query(sql)); + PAssert.that(result).containsInAnyOrder(resultRow1, resultRow2, resultRow3); + pipeline.run().waitUntilFinish(); + } + + @Test + public void testSHA256() throws Exception { + Schema resultType = Schema.builder().addByteArrayField("field").build(); + Row resultRow1 = + Row.withSchema(resultType).addValues(DigestUtils.sha256("foobar".getBytes(UTF_8))).build(); + Row resultRow2 = + Row.withSchema(resultType).addValues(DigestUtils.sha256(" ".getBytes(UTF_8))).build(); + Row resultRow3 = + Row.withSchema(resultType) + .addValues(DigestUtils.sha256("abcABCжщфЖЩФ".getBytes(UTF_8))) + .build(); + String sql = "SELECT SHA256(f_bytes) FROM PCOLLECTION WHERE f_func = 'HashingFn'"; + PCollection result = boundedInputBytes.apply("testUdf", SqlTransform.query(sql)); + PAssert.that(result).containsInAnyOrder(resultRow1, resultRow2, resultRow3); + pipeline.run().waitUntilFinish(); + } + + @Test + public void testSHA512() throws Exception { + Schema resultType = Schema.builder().addByteArrayField("field").build(); + Row resultRow1 = + Row.withSchema(resultType).addValues(DigestUtils.sha512("foobar".getBytes(UTF_8))).build(); + Row resultRow2 = + Row.withSchema(resultType).addValues(DigestUtils.sha512(" ".getBytes(UTF_8))).build(); + Row resultRow3 = + Row.withSchema(resultType) + .addValues(DigestUtils.sha512("abcABCжщфЖЩФ".getBytes(UTF_8))) + .build(); + String sql = "SELECT SHA512(f_bytes) FROM PCOLLECTION WHERE f_func = 'HashingFn'"; + PCollection result = boundedInputBytes.apply("testUdf", SqlTransform.query(sql)); + PAssert.that(result).containsInAnyOrder(resultRow1, resultRow2, resultRow3); + pipeline.run().waitUntilFinish(); + } } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSqlUdfExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSqlUdfExpressionTest.java index 82ffccf9fb6f..495f32941f6f 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSqlUdfExpressionTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/udf/BeamSqlUdfExpressionTest.java @@ -21,6 +21,7 @@ import org.apache.beam.sdk.extensions.sql.integrationtest.BeamSqlBuiltinFunctionsIntegrationTestBase; import org.apache.beam.sdk.schemas.Schema.TypeName; +import org.apache.commons.codec.digest.DigestUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -199,4 +200,44 @@ public void testRightPad() throws Exception { checker.buildRunAndCheck(); } + + @Test + public void testMd5() throws Exception { + ExpressionChecker checker = + new ExpressionChecker() + .addExpr("MD5('foobar')", DigestUtils.md5("foobar")) + .addExpr("MD5('中文')", DigestUtils.md5("中文")) + .addExprWithNullExpectedValue("MD5(CAST(NULL AS VARCHAR(0)))", TypeName.BYTES); + checker.buildRunAndCheck(); + } + + @Test + public void testSHA1() throws Exception { + ExpressionChecker checker = + new ExpressionChecker() + .addExpr("SHA1('foobar')", DigestUtils.sha1("foobar")) + .addExpr("SHA1('中文')", DigestUtils.sha1("中文")) + .addExprWithNullExpectedValue("SHA1(CAST(NULL AS VARCHAR(0)))", TypeName.BYTES); + checker.buildRunAndCheck(); + } + + @Test + public void testSHA256() throws Exception { + ExpressionChecker checker = + new ExpressionChecker() + .addExpr("SHA256('foobar')", DigestUtils.sha256("foobar")) + .addExpr("SHA256('中文')", DigestUtils.sha256("中文")) + .addExprWithNullExpectedValue("SHA256(CAST(NULL AS VARCHAR(0)))", TypeName.BYTES); + checker.buildRunAndCheck(); + } + + @Test + public void testSHA512() throws Exception { + ExpressionChecker checker = + new ExpressionChecker() + .addExpr("SHA512('foobar')", DigestUtils.sha512("foobar")) + .addExpr("SHA512('中文')", DigestUtils.sha512("中文")) + .addExprWithNullExpectedValue("SHA512(CAST(NULL AS VARCHAR(0)))", TypeName.BYTES); + checker.buildRunAndCheck(); + } } diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTest.java index 8385c706315a..c6aa39b1ef37 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTest.java @@ -4780,6 +4780,31 @@ public void testTumbleAsTVF() { pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES)); } + @Test + public void testTVFTumbleAggregation() { + String sql = + "SELECT COUNT(*) as field_count, " + + "window_start " + + "FROM TUMBLE((select * from KeyValue), descriptor(ts), 'INTERVAL 1 SECOND') " + + "GROUP BY window_start"; + ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config); + BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql); + + PCollection stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode); + + final Schema schema = + Schema.builder().addInt64Field("field_count").addDateTimeField("window_start").build(); + PAssert.that(stream) + .containsInAnyOrder( + Row.withSchema(schema) + .addValues(1L, new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC())) + .build(), + Row.withSchema(schema) + .addValues(1L, new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC())) + .build()); + pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES)); + } + @Test public void testIsNullTrueFalse() { String sql = diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java index eedb42c2a05f..3b10b33dcdd9 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java @@ -34,6 +34,7 @@ import org.apache.beam.fn.harness.data.PCollectionConsumerRegistry; import org.apache.beam.fn.harness.data.PTransformFunctionRegistry; import org.apache.beam.fn.harness.state.BeamFnStateClient; +import org.apache.beam.fn.harness.state.StateBackedIterable.StateBackedIterableTranslationContext; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleSplitRequest; import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleSplitRequest.DesiredSplit; @@ -120,6 +121,7 @@ public BeamFnDataReadRunner createRunnerForPTransform( processBundleInstructionId, coders, beamFnDataClient, + beamFnStateClient, addProgressRequestCallback, consumer); startFunctionRegistry.register(pTransformId, runner::registerInputLocation); @@ -149,6 +151,7 @@ public BeamFnDataReadRunner createRunnerForPTransform( Supplier processBundleInstructionIdSupplier, Map coders, BeamFnDataClient beamFnDataClient, + BeamFnStateClient beamFnStateClient, Consumer addProgressRequestCallback, FnDataReceiver> consumer) throws IOException { @@ -163,7 +166,20 @@ public BeamFnDataReadRunner createRunnerForPTransform( RehydratedComponents.forComponents(Components.newBuilder().putAllCoders(coders).build()); this.coder = (Coder>) - CoderTranslation.fromProto(coders.get(port.getCoderId()), components); + CoderTranslation.fromProto( + coders.get(port.getCoderId()), + components, + new StateBackedIterableTranslationContext() { + @Override + public BeamFnStateClient getStateClient() { + return beamFnStateClient; + } + + @Override + public Supplier getCurrentInstructionId() { + return processBundleInstructionIdSupplier; + } + }); addProgressRequestCallback.accept( () -> { diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java index c2570c1c8490..684f39911320 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java @@ -30,6 +30,7 @@ import org.apache.beam.fn.harness.data.PCollectionConsumerRegistry; import org.apache.beam.fn.harness.data.PTransformFunctionRegistry; import org.apache.beam.fn.harness.state.BeamFnStateClient; +import org.apache.beam.fn.harness.state.StateBackedIterable.StateBackedIterableTranslationContext; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnApi.RemoteGrpcPort; import org.apache.beam.model.pipeline.v1.Endpoints; @@ -99,7 +100,12 @@ public BeamFnDataWriteRunner createRunnerForPTransform( BeamFnDataWriteRunner runner = new BeamFnDataWriteRunner<>( - pTransformId, pTransform, processBundleInstructionId, coders, beamFnDataClient); + pTransformId, + pTransform, + processBundleInstructionId, + coders, + beamFnDataClient, + beamFnStateClient); startFunctionRegistry.register(pTransformId, runner::registerForOutput); pCollectionConsumerRegistry.register( getOnlyElement(pTransform.getInputsMap().values()), @@ -124,7 +130,8 @@ public BeamFnDataWriteRunner createRunnerForPTransform( RunnerApi.PTransform remoteWriteNode, Supplier processBundleInstructionIdSupplier, Map coders, - BeamFnDataClient beamFnDataClientFactory) + BeamFnDataClient beamFnDataClientFactory, + BeamFnStateClient beamFnStateClient) throws IOException { this.pTransformId = pTransformId; RemoteGrpcPort port = RemoteGrpcPortWrite.fromPTransform(remoteWriteNode).getPort(); @@ -136,7 +143,20 @@ public BeamFnDataWriteRunner createRunnerForPTransform( RehydratedComponents.forComponents(Components.newBuilder().putAllCoders(coders).build()); this.coder = (Coder>) - CoderTranslation.fromProto(coders.get(port.getCoderId()), components); + CoderTranslation.fromProto( + coders.get(port.getCoderId()), + components, + new StateBackedIterableTranslationContext() { + @Override + public BeamFnStateClient getStateClient() { + return beamFnStateClient; + } + + @Override + public Supplier getCurrentInstructionId() { + return processBundleInstructionIdSupplier; + } + }); } public void registerForOutput() { diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java index 98b3e66ad4f6..0baf76b9dcdf 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java @@ -92,7 +92,7 @@ import org.apache.beam.sdk.transforms.reflect.DoFnInvokers; import org.apache.beam.sdk.transforms.reflect.DoFnSignature; import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration; -import org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerDeclaration; +import org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerFamilyDeclaration; import org.apache.beam.sdk.transforms.reflect.DoFnSignatures; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.HasProgress; @@ -460,14 +460,13 @@ public void accept(WindowedValue input) throws Exception { // Extract out relevant TimerFamilySpec information in preparation for execution. for (Map.Entry entry : parDoPayload.getTimerFamilySpecsMap().entrySet()) { - String timerFamilyId = entry.getKey(); - TimeDomain timeDomain = - DoFnSignatures.getTimerSpecOrThrow( - doFnSignature.timerDeclarations().get(timerFamilyId), doFn) - .getTimeDomain(); + // The timer family spec map key is either from timerId of timer declaration or + // timerFamilyId from timer family declaration. + String timerIdOrTimerFamilyId = entry.getKey(); + TimeDomain timeDomain = translateTimeDomain(entry.getValue().getTimeDomain()); Coder> timerCoder = (Coder) rehydratedComponents.getCoder(entry.getValue().getTimerFamilyCoderId()); - timerFamilyInfosBuilder.put(timerFamilyId, KV.of(timeDomain, timerCoder)); + timerFamilyInfosBuilder.put(timerIdOrTimerFamilyId, KV.of(timeDomain, timerCoder)); } timerFamilyInfos = timerFamilyInfosBuilder.build(); @@ -962,16 +961,27 @@ private HandlesSplits.SplitResult trySplitForElementAndRestriction( .build()); } - private void processTimer(String timerId, TimeDomain timeDomain, Timer timer) { + private void processTimer( + String timerIdOrTimerFamilyId, TimeDomain timeDomain, Timer timer) { currentTimer = timer; currentTimeDomain = timeDomain; onTimerContext = new OnTimerContext<>(timer.getUserKey()); + // The timerIdOrTimerFamilyId contains either a timerId from timer declaration or timerFamilyId + // from timer family declaration. + String timerId = + timerIdOrTimerFamilyId.startsWith(TimerFamilyDeclaration.PREFIX) + ? "" + : timerIdOrTimerFamilyId; + String timerFamilyId = + timerIdOrTimerFamilyId.startsWith(TimerFamilyDeclaration.PREFIX) + ? timerIdOrTimerFamilyId + : ""; try { Iterator windowIterator = (Iterator) timer.getWindows().iterator(); while (windowIterator.hasNext()) { currentWindow = windowIterator.next(); - doFnInvoker.invokeOnTimer(timerId, "", onTimerContext); + doFnInvoker.invokeOnTimer(timerId, timerFamilyId, onTimerContext); } } finally { currentTimer = null; @@ -1037,16 +1047,15 @@ private class FnApiTimer implements org.apache.beam.sdk.state.Timer { BoundedWindow boundedWindow, Instant elementTimestampOrTimerHoldTimestamp, Instant elementTimestampOrTimerFireTimestamp, - PaneInfo paneInfo) { + PaneInfo paneInfo, + TimeDomain timeDomain) { this.timerId = timerId; this.userKey = userKey; this.dynamicTimerTag = dynamicTimerTag; this.elementTimestampOrTimerHoldTimestamp = elementTimestampOrTimerHoldTimestamp; this.boundedWindow = boundedWindow; this.paneInfo = paneInfo; - - TimerDeclaration timerDeclaration = doFnSignature.timerDeclarations().get(timerId); - this.timeDomain = DoFnSignatures.getTimerSpecOrThrow(timerDeclaration, doFn).getTimeDomain(); + this.timeDomain = timeDomain; switch (timeDomain) { case EVENT_TIME: @@ -1207,15 +1216,49 @@ private void output(Instant scheduledTime) { } } - private static class FnApiTimerMap implements TimerMap { - FnApiTimerMap() {} + private class FnApiTimerMap implements TimerMap { + private final String timerFamilyId; + private final K userKey; + private final TimeDomain timeDomain; + private final Instant elementTimestampOrTimerHoldTimestamp; + private final Instant elementTimestampOrTimerFireTimestamp; + private final BoundedWindow boundedWindow; + private final PaneInfo paneInfo; + + FnApiTimerMap( + String timerFamilyId, + K userKey, + BoundedWindow boundedWindow, + Instant elementTimestampOrTimerHoldTimestamp, + Instant elementTimestampOrTimerFireTimestamp, + PaneInfo paneInfo) { + this.timerFamilyId = timerFamilyId; + this.userKey = userKey; + this.elementTimestampOrTimerHoldTimestamp = elementTimestampOrTimerHoldTimestamp; + this.elementTimestampOrTimerFireTimestamp = elementTimestampOrTimerFireTimestamp; + this.boundedWindow = boundedWindow; + this.paneInfo = paneInfo; + this.timeDomain = + translateTimeDomain( + parDoPayload.getTimerFamilySpecsMap().get(timerFamilyId).getTimeDomain()); + } @Override - public void set(String timerId, Instant absoluteTime) {} + public void set(String dynamicTimerTag, Instant absoluteTime) { + get(dynamicTimerTag).set(absoluteTime); + } @Override - public org.apache.beam.sdk.state.Timer get(String timerId) { - return null; + public org.apache.beam.sdk.state.Timer get(String dynamicTimerTag) { + return new FnApiTimer( + timerFamilyId, + userKey, + dynamicTimerTag, + boundedWindow, + elementTimestampOrTimerHoldTimestamp, + elementTimestampOrTimerFireTimestamp, + paneInfo, + timeDomain); } } @@ -1447,6 +1490,8 @@ public org.apache.beam.sdk.state.Timer timer(String timerId) { // For the initial timestamps we pass in the current elements timestamp for the hold timestamp // and the current element's timestamp which will be used for the fire timestamp if this // timer is in the EVENT time domain. + TimeDomain timeDomain = + translateTimeDomain(parDoPayload.getTimerFamilySpecsMap().get(timerId).getTimeDomain()); return new FnApiTimer( timerId, ((KV) currentElement.getValue()).getKey(), @@ -1454,13 +1499,19 @@ public org.apache.beam.sdk.state.Timer timer(String timerId) { currentWindow, currentElement.getTimestamp(), currentElement.getTimestamp(), - currentElement.getPane()); + currentElement.getPane(), + timeDomain); } @Override - public TimerMap timerFamily(String tagId) { - // TODO: implement timerFamily - return null; + public TimerMap timerFamily(String timerFamilyId) { + return new FnApiTimerMap( + timerFamilyId, + ((KV) currentElement.getValue()).getKey(), + currentWindow, + currentElement.getTimestamp(), + currentElement.getTimestamp(), + currentElement.getPane()); } @Override @@ -1682,10 +1733,24 @@ public State state(String stateId, boolean alwaysFetched) { @Override public org.apache.beam.sdk.state.Timer timer(String timerId) { + TimeDomain timeDomain = + translateTimeDomain(parDoPayload.getTimerFamilySpecsMap().get(timerId).getTimeDomain()); return new FnApiTimer( timerId, currentTimer.getUserKey(), - currentTimer.getDynamicTimerTag(), + "", + currentWindow, + currentTimer.getHoldTimestamp(), + currentTimer.getFireTimestamp(), + currentTimer.getPane(), + timeDomain); + } + + @Override + public TimerMap timerFamily(String timerFamilyId) { + return new FnApiTimerMap( + timerFamilyId, + currentTimer.getUserKey(), currentWindow, currentTimer.getHoldTimestamp(), currentTimer.getFireTimestamp(), @@ -1693,9 +1758,9 @@ public org.apache.beam.sdk.state.Timer timer(String timerId) { } @Override - public TimerMap timerFamily(String tagId) { - // TODO: implement timerFamily - return super.timerFamily(tagId); + public String timerId(DoFn doFn) { + // Timer id is aliased to dynamic timer tag in a TimerFamily timer. + return currentTimer.getDynamicTimerTag(); } @Override @@ -1708,4 +1773,18 @@ public String getErrorContext() { return "FnApiDoFnRunner/OnTimer"; } } + + private TimeDomain translateTimeDomain( + org.apache.beam.model.pipeline.v1.RunnerApi.TimeDomain.Enum domain) { + switch (domain) { + case EVENT_TIME: + return TimeDomain.EVENT_TIME; + case PROCESSING_TIME: + return TimeDomain.PROCESSING_TIME; + case SYNCHRONIZED_PROCESSING_TIME: + return TimeDomain.SYNCHRONIZED_PROCESSING_TIME; + default: + throw new IllegalArgumentException("Unknown time domain"); + } + } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java index 48a55bf3e8ae..515f0644c2ef 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java @@ -27,6 +27,7 @@ import org.apache.beam.runners.core.SideInputReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.SdkHarnessOptions; import org.apache.beam.sdk.transforms.Combine.CombineFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; @@ -40,6 +41,10 @@ /** Static utility methods that provide {@link GroupingTable} implementations. */ public class PrecombineGroupingTable implements GroupingTable { + private static long getGroupingTableSizeBytes(PipelineOptions options) { + return options.as(SdkHarnessOptions.class).getGroupingTableMaxSizeMb() * 1024L * 1024L; + } + /** Returns a {@link GroupingTable} that combines inputs into a accumulator. */ public static GroupingTable, InputT, AccumT> combining( PipelineOptions options, @@ -50,7 +55,7 @@ public static GroupingTable, InputT, AccumT new ValueCombiner<>( GlobalCombineFnRunners.create(combineFn), NullSideInputReader.empty(), options); return new PrecombineGroupingTable<>( - DEFAULT_MAX_GROUPING_TABLE_BYTES, + getGroupingTableSizeBytes(options), new WindowingCoderGroupingKeyCreator<>(keyCoder), WindowedPairInfo.create(), valueCombiner, @@ -73,7 +78,7 @@ GroupingTable, InputT, AccumT> combiningAndSampling( new ValueCombiner<>( GlobalCombineFnRunners.create(combineFn), NullSideInputReader.empty(), options); return new PrecombineGroupingTable<>( - DEFAULT_MAX_GROUPING_TABLE_BYTES, + getGroupingTableSizeBytes(options), new WindowingCoderGroupingKeyCreator<>(keyCoder), WindowedPairInfo.create(), valueCombiner, @@ -255,10 +260,6 @@ public OutputT extract(WindowedValue windowedKey, AccumT accumulator) { } } - // By default, how many bytes we allow the grouping table to consume before - // it has to be flushed. - private static final long DEFAULT_MAX_GROUPING_TABLE_BYTES = 100_000_000L; - // How many bytes a word in the JVM has. private static final int BYTES_PER_JVM_WORD = getBytesPerJvmWord(); /** diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java index 38e202700ed0..cb3c1953aa53 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java @@ -77,7 +77,7 @@ public BagUserState( new DataStreams.DataStreamDecoder( valueCoder, DataStreams.inbound( - StateFetchingIterators.forFirstChunk(beamFnStateClient, request)))); + StateFetchingIterators.readAllStartingFrom(beamFnStateClient, request)))); this.newValues = new ArrayList<>(); } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java index d795d44f163f..795561047d1a 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java @@ -71,7 +71,8 @@ public Iterable get() { new DataStreams.DataStreamDecoder( keyCoder, DataStreams.inbound( - StateFetchingIterators.forFirstChunk(beamFnStateClient, requestBuilder.build())))); + StateFetchingIterators.readAllStartingFrom( + beamFnStateClient, requestBuilder.build())))); } @Override @@ -97,6 +98,7 @@ public Iterable get(K k) { new DataStreams.DataStreamDecoder( valueCoder, DataStreams.inbound( - StateFetchingIterators.forFirstChunk(beamFnStateClient, requestBuilder.build())))); + StateFetchingIterators.readAllStartingFrom( + beamFnStateClient, requestBuilder.build())))); } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java new file mode 100644 index 000000000000..94c8f9a8f614 --- /dev/null +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.fn.harness.state; + +import static org.apache.beam.runners.core.construction.ModelCoders.STATE_BACKED_ITERABLE_CODER_URN; + +import com.google.auto.service.AutoService; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; +import org.apache.beam.runners.core.construction.CoderTranslator; +import org.apache.beam.runners.core.construction.CoderTranslatorRegistrar; +import org.apache.beam.sdk.coders.IterableLikeCoder; +import org.apache.beam.sdk.fn.stream.DataStreams; +import org.apache.beam.sdk.util.BufferedElementCountingOutputStream; +import org.apache.beam.sdk.util.VarInt; +import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; + +/** + * A {@link BeamFnStateClient state} backed iterable which allows for fetching elements over the + * portability state API. See remote references for + * additional details. + * + *

One must supply a {@link StateBackedIterableTranslationContext} when using {@link + * CoderTranslator#fromComponents} to be able to create a {@link StateBackedIterable.Coder}. + */ +public class StateBackedIterable implements Iterable { + + private final BeamFnStateClient beamFnStateClient; + private final org.apache.beam.sdk.coders.Coder elemCoder; + @VisibleForTesting final StateRequest request; + @VisibleForTesting final List prefix; + + public StateBackedIterable( + BeamFnStateClient beamFnStateClient, + String instructionId, + ByteString runnerKey, + org.apache.beam.sdk.coders.Coder elemCoder, + List prefix) { + this.beamFnStateClient = beamFnStateClient; + this.elemCoder = elemCoder; + + StateRequest.Builder requestBuilder = StateRequest.newBuilder(); + requestBuilder + .setInstructionId(instructionId) + .getStateKeyBuilder() + .getRunnerBuilder() + .setKey(runnerKey); + this.request = requestBuilder.build(); + this.prefix = prefix; + } + + @Override + public Iterator iterator() { + return Iterators.concat( + prefix.iterator(), + new DataStreams.DataStreamDecoder( + elemCoder, + DataStreams.inbound( + StateFetchingIterators.readAllStartingFrom(beamFnStateClient, request)))); + } + + /** + * Decodes an {@link Iterable} that might be backed by state. If the terminator at the end of the + * value stream is {@code -1} then we return a {@link StateBackedIterable} otherwise we return an + * {@link Iterable}. + */ + public static class Coder extends IterableLikeCoder> { + + private final BeamFnStateClient beamFnStateClient; + private final Supplier instructionId; + + public Coder( + BeamFnStateClient beamFnStateClient, + Supplier instructionId, + org.apache.beam.sdk.coders.Coder elemCoder) { + super(elemCoder, "StateBackedIterable"); + this.beamFnStateClient = beamFnStateClient; + this.instructionId = instructionId; + } + + @Override + protected Iterable decodeToIterable(List decodedElements) { + return decodedElements; + } + + @Override + protected Iterable decodeToIterable( + List decodedElements, long terminatorValue, InputStream in) throws IOException { + if (terminatorValue == -1L) { + long tokenLength = VarInt.decodeLong(in); + ByteString token = ByteString.readFrom(ByteStreams.limit(in, tokenLength)); + return new StateBackedIterable<>( + beamFnStateClient, instructionId.get(), token, getElemCoder(), decodedElements); + } else { + throw new IllegalStateException( + String.format( + "StateBackedIterable expected terminator of 0 or -1 but received %s.", + terminatorValue)); + } + } + + @Override + public void encode(Iterable iterable, OutputStream outStream) throws IOException { + if (!(iterable instanceof StateBackedIterable)) { + super.encode(iterable, outStream); + return; + } + + StateBackedIterable stateBackedIterable = (StateBackedIterable) iterable; + + DataOutputStream dataOutStream = new DataOutputStream(outStream); + // We don't know the size without traversing it so use a fixed size buffer + // and encode as many elements as possible into it before outputting the size followed + // by the elements. + dataOutStream.writeInt(-1); + BufferedElementCountingOutputStream countingOutputStream = + new BufferedElementCountingOutputStream(dataOutStream, -1L); + // Encode only the prefix + for (T elem : stateBackedIterable.prefix) { + countingOutputStream.markElementStart(); + getElemCoder().encode(elem, countingOutputStream); + } + countingOutputStream.finish(); + // Make sure all our output gets pushed to the underlying outStream. + dataOutStream.flush(); + + // Append 'len(token) token' after the -1 stream terminator. + VarInt.encode( + stateBackedIterable.request.getStateKey().getRunner().getKey().size(), outStream); + stateBackedIterable.request.getStateKey().getRunner().getKey().writeTo(outStream); + } + } + + /** Additional parameters required by the {@link StateBackedIterable.Coder}. */ + public interface StateBackedIterableTranslationContext extends TranslationContext { + BeamFnStateClient getStateClient(); + + Supplier getCurrentInstructionId(); + } + + /** A {@link CoderTranslatorRegistrar} for {@code beam:coder:state_backed_iterable:v1}. */ + @AutoService(CoderTranslatorRegistrar.class) + public static class Registrar implements CoderTranslatorRegistrar { + + @Override + public Map, String> getCoderURNs() { + return Collections.singletonMap( + StateBackedIterable.Coder.class, STATE_BACKED_ITERABLE_CODER_URN); + } + + @Override + public Map< + Class, + CoderTranslator> + getCoderTranslators() { + return ImmutableMap.of(StateBackedIterable.Coder.class, new Translator()); + } + } + + /** + * A {@link CoderTranslator} for {@code beam:coder:state_backed_iterable:v1}. + * + *

One must supply a {@link StateBackedIterableTranslationContext} during {@link + * CoderTranslator#fromComponents} to be able to successfully create an instance. + */ + private static class Translator implements CoderTranslator> { + + @Override + public List> getComponents( + StateBackedIterable.Coder from) { + return Collections.>singletonList(from.getElemCoder()); + } + + @Override + public StateBackedIterable.Coder fromComponents( + List> components, + byte[] payload, + TranslationContext context) { + if (context instanceof StateBackedIterableTranslationContext) { + return new StateBackedIterable.Coder<>( + ((StateBackedIterableTranslationContext) context).getStateClient(), + ((StateBackedIterableTranslationContext) context).getCurrentInstructionId(), + Iterables.getOnlyElement(components)); + } else { + throw new IllegalStateException( + String.format( + "Unable to construct coder %s. Expected translation context %s but received %s.", + STATE_BACKED_ITERABLE_CODER_URN, + StateBackedIterableTranslationContext.class.getName(), + context.getClass().getName())); + } + } + } +} diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java index a7cb38d7b25a..7af431de0659 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java @@ -46,7 +46,7 @@ private StateFetchingIterators() {} * only) chunk of a state stream. This state request will be populated with a continuation * token to request further chunks of the stream if required. */ - public static Iterator forFirstChunk( + public static Iterator readAllStartingFrom( BeamFnStateClient beamFnStateClient, StateRequest stateRequestForFirstChunk) { return new LazyBlockingStateFetchingIterator(beamFnStateClient, stateRequestForFirstChunk); } diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java index 75f3105b0b22..9293d011c35a 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java @@ -235,6 +235,7 @@ public void testReuseForMultipleBundles() throws Exception { bundleId::get, COMPONENTS.getCodersMap(), mockBeamFnDataClient, + null /* beamFnStateClient */, (PTransformRunnerFactory.ProgressRequestCallback callback) -> { progressCallbacks.add(callback); }, diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java index 7159801d1eee..a2f4c8f27511 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java @@ -213,7 +213,8 @@ public void testReuseForMultipleBundles() throws Exception { RemoteGrpcPortWrite.writeToPort("myWrite", PORT_SPEC).toPTransform(), bundleId::get, COMPONENTS.getCodersMap(), - mockBeamFnDataClient); + mockBeamFnDataClient, + null /* beamFnStateClient */); // Process for bundle id 0 writeRunner.registerForOutput(); diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java index 1da967269b32..fb00550d253c 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java @@ -59,6 +59,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.Environment; import org.apache.beam.runners.core.construction.CoderTranslation; +import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.runners.core.construction.ParDoTranslation; import org.apache.beam.runners.core.construction.PipelineTranslation; @@ -94,6 +95,7 @@ import org.apache.beam.sdk.state.StateSpecs; import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.state.Timer; +import org.apache.beam.sdk.state.TimerMap; import org.apache.beam.sdk.state.TimerSpec; import org.apache.beam.sdk.state.TimerSpecs; import org.apache.beam.sdk.state.ValueState; @@ -759,58 +761,6 @@ public void testUsingMetrics() throws Exception { assertThat(result, containsInAnyOrder(expected.toArray())); } - private static class TestTimerfulDoFn extends DoFn, String> { - @StateId("bag") - private final StateSpec> bagStateSpec = StateSpecs.bag(StringUtf8Coder.of()); - - @TimerId("event") - private final TimerSpec eventTimerSpec = TimerSpecs.timer(TimeDomain.EVENT_TIME); - - @TimerId("processing") - private final TimerSpec processingTimerSpec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME); - - @ProcessElement - public void processElement( - ProcessContext context, - @StateId("bag") BagState bagState, - @TimerId("event") Timer eventTimeTimer, - @TimerId("processing") Timer processingTimeTimer) { - context.output("main" + context.element().getKey() + Iterables.toString(bagState.read())); - bagState.add(context.element().getValue()); - eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(1L)); - processingTimeTimer.offset(Duration.millis(2L)); - processingTimeTimer.setRelative(); - } - - @OnTimer("event") - public void eventTimer( - OnTimerContext context, - @StateId("bag") BagState bagState, - @TimerId("event") Timer eventTimeTimer, - @TimerId("processing") Timer processingTimeTimer) { - context.output("event" + Iterables.toString(bagState.read())); - bagState.add("event"); - eventTimeTimer - .withOutputTimestamp(context.timestamp()) - .set(context.fireTimestamp().plus(11L)); - processingTimeTimer.offset(Duration.millis(12L)); - processingTimeTimer.setRelative(); - } - - @OnTimer("processing") - public void processingTimer( - OnTimerContext context, - @StateId("bag") BagState bagState, - @TimerId("event") Timer eventTimeTimer, - @TimerId("processing") Timer processingTimeTimer) { - context.output("processing" + Iterables.toString(bagState.read())); - bagState.add("processing"); - eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(21L)); - processingTimeTimer.offset(Duration.millis(22L)); - processingTimeTimer.setRelative(); - } - } - @Test public void testTimers() throws Exception { dateTimeProvider.setDateTimeFixed(10000L); @@ -889,6 +839,10 @@ public void testTimers() throws Exception { LogicalEndpoint eventTimer = LogicalEndpoint.timer("57L", TEST_TRANSFORM_ID, "ts-event"); LogicalEndpoint processingTimer = LogicalEndpoint.timer("57L", TEST_TRANSFORM_ID, "ts-processing"); + LogicalEndpoint eventFamilyTimer = + LogicalEndpoint.timer("57L", TEST_TRANSFORM_ID, "tfs-event-family"); + LogicalEndpoint processingFamilyTimer = + LogicalEndpoint.timer("57L", TEST_TRANSFORM_ID, "tfs-processing-family"); // Ensure that bag user state that is initially empty or populated works. // Ensure that the key order does not matter when we traverse over KV pairs. FnDataReceiver> mainInput = @@ -909,6 +863,13 @@ public void testTimers() throws Exception { processingTimer, timerInGlobalWindow("C", new Instant(1800L), new Instant(2800L))); fakeTimerClient.sendTimer( processingTimer, timerInGlobalWindow("B", new Instant(1900L), new Instant(2900L))); + fakeTimerClient.sendTimer( + eventFamilyTimer, + dynamicTimerInGlobalWindow("B", "event-timer2", new Instant(2000L), new Instant(3000L))); + fakeTimerClient.sendTimer( + processingFamilyTimer, + dynamicTimerInGlobalWindow( + "Y", "processing-timer2", new Instant(2100L), new Instant(3100L))); assertThat( mainOutputValues, contains( @@ -921,7 +882,9 @@ public void testTimers() throws Exception { timestampedValueInGlobalWindow("event[A0, event]", new Instant(1600L)), timestampedValueInGlobalWindow("processing[X0, X1, X2]", new Instant(1700L)), timestampedValueInGlobalWindow("processing[C0]", new Instant(1800L)), - timestampedValueInGlobalWindow("processing[event]", new Instant(1900L)))); + timestampedValueInGlobalWindow("processing[event]", new Instant(1900L)), + timestampedValueInGlobalWindow("event-family[event, processing]", new Instant(2000L)), + timestampedValueInGlobalWindow("processing-family[Y1, Y2]", new Instant(2100L)))); assertThat( fakeTimerClient.getTimers(eventTimer), contains( @@ -934,7 +897,9 @@ public void testTimers() throws Exception { timerInGlobalWindow("A", new Instant(1600L), new Instant(2611L)), timerInGlobalWindow("X", new Instant(1700L), new Instant(1721L)), timerInGlobalWindow("C", new Instant(1800L), new Instant(1821L)), - timerInGlobalWindow("B", new Instant(1900L), new Instant(1921L)))); + timerInGlobalWindow("B", new Instant(1900L), new Instant(1921L)), + timerInGlobalWindow("B", new Instant(2000L), new Instant(2031L)), + timerInGlobalWindow("Y", new Instant(2100L), new Instant(2141L)))); assertThat( fakeTimerClient.getTimers(processingTimer), contains( @@ -947,19 +912,70 @@ public void testTimers() throws Exception { timerInGlobalWindow("A", new Instant(1600L), new Instant(10012L)), timerInGlobalWindow("X", new Instant(1700L), new Instant(10022L)), timerInGlobalWindow("C", new Instant(1800L), new Instant(10022L)), - timerInGlobalWindow("B", new Instant(1900L), new Instant(10022L)))); + timerInGlobalWindow("B", new Instant(1900L), new Instant(10022L)), + timerInGlobalWindow("B", new Instant(2000L), new Instant(10032L)), + timerInGlobalWindow("Y", new Instant(2100L), new Instant(10042L)))); + assertThat( + fakeTimerClient.getTimers(eventFamilyTimer), + contains( + dynamicTimerInGlobalWindow("X", "event-timer1", new Instant(1000L), new Instant(1003L)), + dynamicTimerInGlobalWindow("Y", "event-timer1", new Instant(1100L), new Instant(1103L)), + dynamicTimerInGlobalWindow("X", "event-timer1", new Instant(1200L), new Instant(1203L)), + dynamicTimerInGlobalWindow("Y", "event-timer1", new Instant(1300L), new Instant(1303L)), + dynamicTimerInGlobalWindow("A", "event-timer1", new Instant(1400L), new Instant(2413L)), + dynamicTimerInGlobalWindow("B", "event-timer1", new Instant(1500L), new Instant(2513L)), + dynamicTimerInGlobalWindow("A", "event-timer1", new Instant(1600L), new Instant(2613L)), + dynamicTimerInGlobalWindow("X", "event-timer1", new Instant(1700L), new Instant(1723L)), + dynamicTimerInGlobalWindow("C", "event-timer1", new Instant(1800L), new Instant(1823L)), + dynamicTimerInGlobalWindow("B", "event-timer1", new Instant(1900L), new Instant(1923L)), + dynamicTimerInGlobalWindow("B", "event-timer1", new Instant(2000L), new Instant(2033L)), + dynamicTimerInGlobalWindow( + "Y", "event-timer1", new Instant(2100L), new Instant(2143L)))); + assertThat( + fakeTimerClient.getTimers(processingFamilyTimer), + contains( + dynamicTimerInGlobalWindow( + "X", "processing-timer1", new Instant(1000L), new Instant(10004L)), + dynamicTimerInGlobalWindow( + "Y", "processing-timer1", new Instant(1100L), new Instant(10004L)), + dynamicTimerInGlobalWindow( + "X", "processing-timer1", new Instant(1200L), new Instant(10004L)), + dynamicTimerInGlobalWindow( + "Y", "processing-timer1", new Instant(1300L), new Instant(10004L)), + dynamicTimerInGlobalWindow( + "A", "processing-timer1", new Instant(1400L), new Instant(10014L)), + dynamicTimerInGlobalWindow( + "B", "processing-timer1", new Instant(1500L), new Instant(10014L)), + dynamicTimerInGlobalWindow( + "A", "processing-timer1", new Instant(1600L), new Instant(10014L)), + dynamicTimerInGlobalWindow( + "X", "processing-timer1", new Instant(1700L), new Instant(10024L)), + dynamicTimerInGlobalWindow( + "C", "processing-timer1", new Instant(1800L), new Instant(10024L)), + dynamicTimerInGlobalWindow( + "B", "processing-timer1", new Instant(1900L), new Instant(10024L)), + dynamicTimerInGlobalWindow( + "B", "processing-timer1", new Instant(2000L), new Instant(10034L)), + dynamicTimerInGlobalWindow( + "Y", "processing-timer1", new Instant(2100L), new Instant(10044L)))); mainOutputValues.clear(); assertFalse(fakeTimerClient.isOutboundClosed(eventTimer)); assertFalse(fakeTimerClient.isOutboundClosed(processingTimer)); + assertFalse(fakeTimerClient.isOutboundClosed(eventFamilyTimer)); + assertFalse(fakeTimerClient.isOutboundClosed(processingFamilyTimer)); fakeTimerClient.closeInbound(eventTimer); fakeTimerClient.closeInbound(processingTimer); + fakeTimerClient.closeInbound(eventFamilyTimer); + fakeTimerClient.closeInbound(processingFamilyTimer); Iterables.getOnlyElement(finishFunctionRegistry.getFunctions()).run(); assertThat(mainOutputValues, empty()); assertTrue(fakeTimerClient.isOutboundClosed(eventTimer)); assertTrue(fakeTimerClient.isOutboundClosed(processingTimer)); + assertTrue(fakeTimerClient.isOutboundClosed(eventFamilyTimer)); + assertTrue(fakeTimerClient.isOutboundClosed(processingFamilyTimer)); Iterables.getOnlyElement(teardownFunctions).run(); assertThat(mainOutputValues, empty()); @@ -967,29 +983,154 @@ public void testTimers() throws Exception { assertEquals( ImmutableMap.builder() .put(bagUserStateKey("bag", "X"), encode("X0", "X1", "X2", "processing")) - .put(bagUserStateKey("bag", "Y"), encode("Y1", "Y2")) + .put(bagUserStateKey("bag", "Y"), encode("Y1", "Y2", "processing-family")) .put(bagUserStateKey("bag", "A"), encode("A0", "event", "event")) - .put(bagUserStateKey("bag", "B"), encode("event", "processing")) + .put(bagUserStateKey("bag", "B"), encode("event", "processing", "event-family")) .put(bagUserStateKey("bag", "C"), encode("C0", "processing")) .build(), fakeStateClient.getData()); } - private WindowedValue valueInWindow(T value, BoundedWindow window) { - return WindowedValue.of(value, window.maxTimestamp(), window, PaneInfo.NO_FIRING); - } - private org.apache.beam.runners.core.construction.Timer timerInGlobalWindow( K userKey, Instant holdTimestamp, Instant fireTimestamp) { + return dynamicTimerInGlobalWindow(userKey, "", holdTimestamp, fireTimestamp); + } + + private org.apache.beam.runners.core.construction.Timer dynamicTimerInGlobalWindow( + K userKey, String dynamicTimerTag, Instant holdTimestamp, Instant fireTimestamp) { return org.apache.beam.runners.core.construction.Timer.of( userKey, - "", + dynamicTimerTag, Collections.singletonList(GlobalWindow.INSTANCE), fireTimestamp, holdTimestamp, PaneInfo.NO_FIRING); } + private WindowedValue valueInWindow(T value, BoundedWindow window) { + return WindowedValue.of(value, window.maxTimestamp(), window, PaneInfo.NO_FIRING); + } + + private static class TestTimerfulDoFn extends DoFn, String> { + @StateId("bag") + private final StateSpec> bagStateSpec = StateSpecs.bag(StringUtf8Coder.of()); + + @TimerId("event") + private final TimerSpec eventTimerSpec = TimerSpecs.timer(TimeDomain.EVENT_TIME); + + @TimerId("processing") + private final TimerSpec processingTimerSpec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME); + + @TimerFamily("event-family") + private final TimerSpec eventTimerFamilySpec = TimerSpecs.timerMap(TimeDomain.EVENT_TIME); + + @TimerFamily("processing-family") + private final TimerSpec processingTimerFamilySpec = + TimerSpecs.timerMap(TimeDomain.PROCESSING_TIME); + + @ProcessElement + public void processElement( + ProcessContext context, + @StateId("bag") BagState bagState, + @TimerId("event") Timer eventTimeTimer, + @TimerId("processing") Timer processingTimeTimer, + @TimerFamily("event-family") TimerMap eventTimerFamily, + @TimerFamily("processing-family") TimerMap processingTimerFamily) { + context.output("main" + context.element().getKey() + Iterables.toString(bagState.read())); + bagState.add(context.element().getValue()); + eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(1L)); + processingTimeTimer.offset(Duration.millis(2L)); + processingTimeTimer.setRelative(); + eventTimerFamily + .get("event-timer1") + .withOutputTimestamp(context.timestamp()) + .set(context.timestamp().plus(3L)); + processingTimerFamily.get("processing-timer1").offset(Duration.millis(4L)).setRelative(); + } + + @OnTimer("event") + public void eventTimer( + OnTimerContext context, + @StateId("bag") BagState bagState, + @TimerId("event") Timer eventTimeTimer, + @TimerId("processing") Timer processingTimeTimer, + @TimerFamily("event-family") TimerMap eventTimerFamily, + @TimerFamily("processing-family") TimerMap processingTimerFamily) { + context.output("event" + Iterables.toString(bagState.read())); + bagState.add("event"); + eventTimeTimer + .withOutputTimestamp(context.timestamp()) + .set(context.fireTimestamp().plus(11L)); + processingTimeTimer.offset(Duration.millis(12L)); + processingTimeTimer.setRelative(); + eventTimerFamily + .get("event-timer1") + .withOutputTimestamp(context.timestamp()) + .set(context.fireTimestamp().plus(13L)); + processingTimerFamily.get("processing-timer1").offset(Duration.millis(14L)).setRelative(); + } + + @OnTimer("processing") + public void processingTimer( + OnTimerContext context, + @StateId("bag") BagState bagState, + @TimerId("event") Timer eventTimeTimer, + @TimerId("processing") Timer processingTimeTimer, + @TimerFamily("event-family") TimerMap eventTimerFamily, + @TimerFamily("processing-family") TimerMap processingTimerFamily) { + context.output("processing" + Iterables.toString(bagState.read())); + bagState.add("processing"); + eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(21L)); + processingTimeTimer.offset(Duration.millis(22L)); + processingTimeTimer.setRelative(); + eventTimerFamily + .get("event-timer1") + .withOutputTimestamp(context.timestamp()) + .set(context.timestamp().plus(23L)); + processingTimerFamily.get("processing-timer1").offset(Duration.millis(24L)).setRelative(); + } + + @OnTimerFamily("event-family") + public void eventFamilyOnTimer( + OnTimerContext context, + @StateId("bag") BagState bagState, + @TimerId("event") Timer eventTimeTimer, + @TimerId("processing") Timer processingTimeTimer, + @TimerFamily("event-family") TimerMap eventTimerFamily, + @TimerFamily("processing-family") TimerMap processingTimerFamily) { + context.output("event-family" + Iterables.toString(bagState.read())); + bagState.add("event-family"); + eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(31L)); + processingTimeTimer.offset(Duration.millis(32L)); + processingTimeTimer.setRelative(); + eventTimerFamily + .get("event-timer1") + .withOutputTimestamp(context.timestamp()) + .set(context.timestamp().plus(33L)); + processingTimerFamily.get("processing-timer1").offset(Duration.millis(34L)).setRelative(); + } + + @OnTimerFamily("processing-family") + public void processingFamilyOnTimer( + OnTimerContext context, + @StateId("bag") BagState bagState, + @TimerId("event") Timer eventTimeTimer, + @TimerId("processing") Timer processingTimeTimer, + @TimerFamily("event-family") TimerMap eventTimerFamily, + @TimerFamily("processing-family") TimerMap processingTimerFamily) { + context.output("processing-family" + Iterables.toString(bagState.read())); + bagState.add("processing-family"); + eventTimeTimer.withOutputTimestamp(context.timestamp()).set(context.timestamp().plus(41L)); + processingTimeTimer.offset(Duration.millis(42L)); + processingTimeTimer.setRelative(); + eventTimerFamily + .get("event-timer1") + .withOutputTimestamp(context.timestamp()) + .set(context.timestamp().plus(43L)); + processingTimerFamily.get("processing-timer1").offset(Duration.millis(44L)).setRelative(); + } + } + /** * Produces a multimap side input {@link StateKey} for the test PTransform id in the global * window. @@ -1188,7 +1329,8 @@ public void testProcessElementForSizedElementAndRestriction() throws Exception { WindowedValue.getFullCoder( CoderTranslation.fromProto( pProto.getComponents().getCodersOrThrow(inputPCollection.getCoderId()), - rehydratedComponents), + rehydratedComponents, + TranslationContext.DEFAULT), (Coder) CoderTranslation.fromProto( pProto @@ -1199,7 +1341,8 @@ public void testProcessElementForSizedElementAndRestriction() throws Exception { .getWindowingStrategiesOrThrow( inputPCollection.getWindowingStrategyId()) .getWindowCoderId()), - rehydratedComponents)); + rehydratedComponents, + TranslationContext.DEFAULT)); String outputPCollectionId = pTransform.getOutputsOrThrow("output"); ImmutableMap stateData = diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java new file mode 100644 index 000000000000..fc6549b294a2 --- /dev/null +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.fn.harness.state; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.junit.runners.Parameterized; + +/** Tests for {@link StateBackedIterable}. */ +@RunWith(Enclosed.class) +public class StateBackedIterableTest { + + @RunWith(Parameterized.class) + public static class IterationTest { + @Parameterized.Parameters + public static Iterable data() { + return ImmutableList.builder() + .add(new Object[] {Collections.emptyList(), "emptySuffix", ImmutableList.of()}) + .add(new Object[] {ImmutableList.of("A", "B"), "emptySuffix", ImmutableList.of("A", "B")}) + .add( + new Object[] { + Collections.emptyList(), + "nonEmptySuffix", + ImmutableList.of("C", "D", "E", "F", "G", "H", "I", "J", "K") + }) + .add( + new Object[] { + ImmutableList.of("A", "B"), + "nonEmptySuffix", + ImmutableList.of("A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K") + }) + .build(); + } + + @Parameterized.Parameter(0) + public List prefix; + + @Parameterized.Parameter(1) + public String suffixKey; + + @Parameterized.Parameter(2) + public List expected; + + @Test + public void testReiteration() throws Exception { + FakeBeamFnStateClient fakeBeamFnStateClient = + new FakeBeamFnStateClient( + ImmutableMap.of( + key("nonEmptySuffix"), encode("C", "D", "E", "F", "G", "H", "I", "J", "K"), + key("emptySuffix"), encode())); + + StateBackedIterable iterable = + new StateBackedIterable<>( + fakeBeamFnStateClient, + "instruction", + encode(suffixKey), + StringUtf8Coder.of(), + prefix); + + assertEquals(expected, Lists.newArrayList(iterable)); + assertEquals(expected, Lists.newArrayList(iterable)); + assertEquals(expected, Lists.newArrayList(iterable)); + } + + @Test + public void testUsingInterleavedReiteration() throws Exception { + FakeBeamFnStateClient fakeBeamFnStateClient = + new FakeBeamFnStateClient( + ImmutableMap.of( + key("nonEmptySuffix"), encode("C", "D", "E", "F", "G", "H", "I", "J", "K"), + key("emptySuffix"), encode())); + + StateBackedIterable iterable = + new StateBackedIterable<>( + fakeBeamFnStateClient, + "instruction", + encode(suffixKey), + StringUtf8Coder.of(), + prefix); + + List> iterators = new ArrayList<>(); + List> results = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + iterators.add(iterable.iterator()); + results.add(new ArrayList<>()); + } + + Random random = new Random(42L); + while (!iterators.isEmpty()) { + int current = random.nextInt(iterators.size()); + if (!iterators.get(current).hasNext()) { + iterators.remove(current); + assertEquals(expected, results.remove(current)); + } else { + results.get(current).add(iterators.get(current).next()); + } + } + } + } + + @RunWith(JUnit4.class) + public static class CoderTest { + @Test + public void testDecodeEncodeRegularIterable() throws Exception { + Iterable iterable = FluentIterable.of("A", "B", "C"); + StateBackedIterable.Coder coder = + new StateBackedIterable.Coder<>(null, () -> "instructionId", StringUtf8Coder.of()); + + // We can't rely on CoderProperties since it requires serialization of the coder + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + coder.encode(iterable, baos); + Iterable result = coder.decode(new ByteArrayInputStream(baos.toByteArray())); + + assertEquals(Lists.newArrayList(iterable), Lists.newArrayList(result)); + } + + @Test + public void testEncodeDecodeStateBackedIterable() throws Exception { + StateBackedIterable iterable = + new StateBackedIterable( + null, "instructionId", encode("key"), StringUtf8Coder.of(), Arrays.asList("A", "B")); + StateBackedIterable.Coder coder = + new StateBackedIterable.Coder<>(null, () -> "instructionId", StringUtf8Coder.of()); + + // We can't rely on CoderProperties since it requires serialization of the coder + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + coder.encode(iterable, baos); + StateBackedIterable result = + (StateBackedIterable) coder.decode(new ByteArrayInputStream(baos.toByteArray())); + + // Ensure that when we round trip the iterable we correctly convert it back to a state backed + // iterable + assertEquals(iterable.prefix, result.prefix); + assertEquals(iterable.request, result.request); + } + } + + private static StateKey key(String id) throws IOException { + return StateKey.newBuilder().setRunner(StateKey.Runner.newBuilder().setKey(encode(id))).build(); + } + + private static ByteString encode(String... values) throws IOException { + ByteString.Output out = ByteString.newOutput(); + for (String value : values) { + StringUtf8Coder.of().encode(value, out); + } + return out.toByteString(); + } +} diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle index 442479563f69..d49cd8e19fde 100644 --- a/sdks/java/io/google-cloud-platform/build.gradle +++ b/sdks/java/io/google-cloud-platform/build.gradle @@ -56,11 +56,13 @@ dependencies { compile library.java.google_http_client compile library.java.google_http_client_jackson2 compile library.java.grpc_all + compile library.java.grpc_alts compile library.java.grpc_auth compile library.java.grpc_core compile library.java.grpc_context compile library.java.grpc_grpclb compile library.java.grpc_netty + compile library.java.grpc_netty_shaded compile library.java.grpc_stub compile library.java.grpc_google_cloud_pubsub_v1 compile library.java.guava diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java index f76a4328dfcb..d460c5d3ca40 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java @@ -288,6 +288,11 @@ public BoundedSource splitAtFraction(double fraction) { source.stream.getName(), fraction); + if (fraction <= 0.0 || fraction >= 1.0) { + LOG.info("BigQuery Storage API does not support splitting at fraction {}", fraction); + return null; + } + SplitReadStreamRequest splitRequest = SplitReadStreamRequest.newBuilder() .setOriginalStream(source.stream) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java index 13b696edd74f..070155ed2162 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.healthcare; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.databind.ObjectMapper; @@ -62,6 +63,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Wait; import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; @@ -130,10 +132,18 @@ * FhirIO.Write.Result#getFailedBodies()} to retrieve a {@link PCollection} of {@link * HealthcareIOError} containing the {@link String} that failed to be ingested and the * exception. + *

Conditional Creating / Updating Resources

+ * {@link FhirIO} supports interfaces for conditional update. These can be useful to handle + * scenarios where an executeBundle failed. For example if you tried to create a resource that + * already exists you can grab the faield bodies of your {@link FhirIO.ExecuteBundles} transform + * with {@link FhirIO.Write.Result#getFailedBodies()} perform some logic on the reason for + * failures and if appropriate route this to {@link FhirIO.ConditionalUpdate} or {@link + * FhirIO.CreateResources} to take the appropriate action on your FHIR store. *

Example *

{@code
  * Pipeline pipeline = ...
  *
+ * // [Begin Reading]
  * // Tail the FHIR store by retrieving resources based on Pub/Sub notifications.
  * FhirIO.Read.Result readResult = p
  *   .apply("Read FHIR notifications",
@@ -143,8 +153,11 @@
  * // happily retrived messages
  * PCollection resources = readResult.getResources();
  * // message IDs that couldn't be retrieved + error context
+ *
  * PCollection> failedReads = readResult.getFailedReads();
+ * // [End Reading]
  *
+ * // [Beign Writing]
  * failedReads.apply("Write Message IDs / Stacktrace for Failed Reads to BigQuery",
  *     BigQueryIO
  *         .write()
@@ -155,17 +168,53 @@
  * FhirIO.Write.Result writeResult =
  *     output.apply("Execute FHIR Bundles", FhirIO.executeBundles(options.getExistingFhirStore()));
  *
+ * // Alternatively you could use import for high throughput to a new store.
+ * FhirIO.Write.Result writeResult =
+ *     output.apply("Import FHIR Resources", FhirIO.executeBundles(options.getNewFhirStore()));
+ * // [End Writing ]
+ *
  * PCollection> failedBundles = writeResult.getFailedInsertsWithErr();
  *
+ * // [Begin Writing to Dead Letter Queue]
  * failedBundles.apply("Write failed bundles to BigQuery",
  *     BigQueryIO
  *         .write()
  *         .to(option.getBQFhirExecuteBundlesDeadLetterTable())
  *         .withFormatFunction(new HealthcareIOErrorToTableRow()));
+ * // [End Writing to Dead Letter Queue]
+ *
+ * // Alternatively you may want to handle DeadLetter with conditional update
+ * // [Begin Reconciliation with Conditional Update]
+ * failedBundles
+ *     .apply("Reconcile with Conditional Update",
+ *         FhirIO.ConditionalUpdate(fhirStore)
+ *             .withFormatBodyFunction(HealthcareIOError::getDataResource)
+ *             .withTypeFunction((HealthcareIOError err) -> {
+ *               String body = err.getDataResource();
+ *               // TODO(user) insert logic to exctract type.
+ *               return params;
+ *             })
+ *             .withSearchParametersFunction((HealthcareIOError err) -> {
+ *               String body = err.getDataResource();
+ *               Map params = new HashMap();
+ *               // TODO(user) insert logic to exctract search query parameters.
+ *               return params;
+ *             });
+ * // [End Reconciliation with Conditional Update]
+ *
+ * // Alternatively you may want to handle DeadLetter with conditional create
+ * // [Begin Reconciliation with Conditional Update]
+ * failedBundles
+ *     .apply("Reconcile with Conditional Create",
+ *         FhirIO.CreateResources(fhirStore)
+ *             .withFormatBodyFunction(HealthcareIOError::getDataResource)
+ *             .withIfNotExistsFunction((HealthcareIOError err) -> {
+ *               String body = err.getDataResource();
+ *               // TODO(user) insert logic to exctract a query to be used in If-Not-Exists header.
+ *               return params;
+ *             });
+ * // [End Reconciliation with Conditional Update]
  *
- * // Alternatively you could use import for high throughput to a new store.
- * FhirIO.Write.Result writeResult =
- *     output.apply("Import FHIR Resources", FhirIO.executeBundles(options.getNewFhirStore()));
  * }***
  * 
*/ @@ -435,6 +484,7 @@ public static class Result implements POutput { * Creates a {@link FhirIO.Write.Result} in the given {@link Pipeline}. @param pipeline the * pipeline * + * @param pipeline the pipeline * @param failedBodies the failed inserts * @return the result */ @@ -442,6 +492,14 @@ static Result in(Pipeline pipeline, PCollection> faile return new Result(pipeline, failedBodies, null); } + /** + * In result. + * + * @param pipeline the pipeline + * @param failedBodies the failed bodies + * @param failedFiles the failed files + * @return the result + */ static Result in( Pipeline pipeline, PCollection> failedBodies, @@ -490,7 +548,9 @@ private Result( if (failedFiles == null) { failedFiles = (PCollection>) - pipeline.apply(Create.empty(HealthcareIOErrorCoder.of(StringUtf8Coder.of()))); + pipeline.apply( + org.apache.beam.sdk.transforms.Create.empty( + HealthcareIOErrorCoder.of(StringUtf8Coder.of()))); } this.failedFiles = failedFiles; } @@ -588,7 +648,7 @@ private static Write.Builder write(String fhirStore) { } /** - * Create Method creates a single FHIR resource. @see * * @param fhirStore the hl 7 v 2 store @@ -611,6 +671,14 @@ public static Write fhirStoresImport( .build(); } + /** + * Fhir stores import write. + * + * @param fhirStore the fhir store + * @param gcsDeadLetterPath the gcs dead letter path + * @param contentStructure the content structure + * @return the write + */ public static Write fhirStoresImport( String fhirStore, String gcsDeadLetterPath, @@ -623,6 +691,15 @@ public static Write fhirStoresImport( .build(); } + /** + * Fhir stores import write. + * + * @param fhirStore the fhir store + * @param gcsTempPath the gcs temp path + * @param gcsDeadLetterPath the gcs dead letter path + * @param contentStructure the content structure + * @return the write + */ public static Write fhirStoresImport( ValueProvider fhirStore, ValueProvider gcsTempPath, @@ -737,6 +814,13 @@ public static class Import extends Write { } } + /** + * Instantiates a new Import. + * + * @param fhirStore the fhir store + * @param deadLetterGcsPath the dead letter gcs path + * @param contentStructure the content structure + */ Import( ValueProvider fhirStore, ValueProvider deadLetterGcsPath, @@ -749,6 +833,7 @@ public static class Import extends Write { this.contentStructure = contentStructure; } } + /** * Instantiates a new Import. * @@ -949,6 +1034,7 @@ public void initFile() throws IOException { * Add to batch. * * @param context the context + * @param window the window * @throws IOException the io exception */ @ProcessElement @@ -998,6 +1084,14 @@ static class ImportFn private HealthcareApiClient client; private final ValueProvider fhirStore; + /** + * Instantiates a new Import fn. + * + * @param fhirStore the fhir store + * @param tempGcsPath the temp gcs path + * @param deadLetterGcsPath the dead letter gcs path + * @param contentStructure the content structure + */ ImportFn( ValueProvider fhirStore, ValueProvider tempGcsPath, @@ -1013,6 +1107,11 @@ static class ImportFn } } + /** + * Init. + * + * @throws IOException the io exception + */ @Setup public void init() throws IOException { tempDir = @@ -1026,6 +1125,10 @@ public void init() throws IOException { /** * Move files to a temporary subdir (to provide common prefix) to execute import with single * GCS URI. + * + * @param element the element + * @param output the output + * @throws IOException the io exception */ @ProcessElement public void importBatch( @@ -1173,4 +1276,339 @@ public void executeBundles(ProcessContext context) { } } } + + /** + * Create resources fhir io . create resources. + * + * @param the type parameter + * @param fhirStore the fhir store + * @return the fhir io . create resources + */ + public static FhirIO.CreateResources createResources(ValueProvider fhirStore) { + return new CreateResources(fhirStore); + } + + /** + * Create resources fhir io . create resources. + * + * @param the type parameter + * @param fhirStore the fhir store + * @return the fhir io . create resources + */ + public static FhirIO.CreateResources createResources(String fhirStore) { + return new CreateResources(fhirStore); + } + /** + * {@link PTransform} for Creating FHIR resources. + * + *

https://cloud.google.com/healthcare/docs/reference/rest/v1beta1/projects.locations.datasets.fhirStores.fhir/create + */ + public static class CreateResources extends PTransform, Write.Result> { + private final String fhirStore; + private SerializableFunction ifNoneExistFunction; + private SerializableFunction formatBodyFunction; + private SerializableFunction typeFunction; + private static final Logger LOG = LoggerFactory.getLogger(CreateResources.class); + + /** + * Instantiates a new Create resources transform. + * + * @param fhirStore the fhir store + */ + CreateResources(ValueProvider fhirStore) { + this.fhirStore = fhirStore.get(); + } + + /** + * Instantiates a new Create resources. + * + * @param fhirStore the fhir store + */ + CreateResources(String fhirStore) { + this.fhirStore = fhirStore; + } + + /** + * This adds a {@link SerializableFunction} that reads an resource string and extracts an + * If-None-Exists query for conditional create. Typically this will just be extracting an ID to + * look for. + * + *

https://cloud.google.com/healthcare/docs/reference/rest/v1beta1/projects.locations.datasets.fhirStores.fhir/create + * + * @param ifNoneExistFunction the if none exist function + * @return the create resources + */ + public CreateResources withIfNotExistFunction( + SerializableFunction ifNoneExistFunction) { + this.ifNoneExistFunction = ifNoneExistFunction; + return this; + } + + /** + * This adds a {@link SerializableFunction} that reads an resource string and extracts an + * resource type. + * + *

https://cloud.google.com/healthcare/docs/reference/rest/v1beta1/projects.locations.datasets.fhirStores.fhir/create + * + * @param typeFunction for extracting type from a resource. + * @return the create resources + */ + public CreateResources withTypeFunction(SerializableFunction typeFunction) { + this.typeFunction = typeFunction; + return this; + } + /** + * With format body function create resources. + * + * @param formatBodyFunction the format body function + * @return the create resources + */ + public CreateResources withFormatBodyFunction( + SerializableFunction formatBodyFunction) { + this.formatBodyFunction = formatBodyFunction; + return this; + } + + @Override + public FhirIO.Write.Result expand(PCollection input) { + checkArgument( + typeFunction != null, + "FhirIO.CreateResources should always be called with a " + "withTypeFunction"); + checkArgument( + formatBodyFunction != null, + "FhirIO.CreateResources should always be called with a " + "withFromatBodyFunction"); + + if (ifNoneExistFunction == null) { + LOG.info("ifNoneExistFunction was null will create resources unconditionally"); + } + + return Write.Result.in( + input.getPipeline(), + input + .apply( + ParDo.of( + new CreateFn( + fhirStore, typeFunction, formatBodyFunction, ifNoneExistFunction))) + .setCoder(HealthcareIOErrorCoder.of(StringUtf8Coder.of()))); + } + + static class CreateFn extends DoFn> { + + private Counter failedBundles = Metrics.counter(CreateFn.class, "failed-bundles"); + private transient HealthcareApiClient client; + private final ObjectMapper mapper = new ObjectMapper(); + private final String fhirStore; + private SerializableFunction ifNoneExistFunction; + private SerializableFunction formatBodyFunction; + private SerializableFunction typeFunction; + + CreateFn( + String fhirStore, + SerializableFunction typeFunction, + SerializableFunction formatBodyFunction, + @Nullable SerializableFunction ifNoneExistFunction) { + this.fhirStore = fhirStore; + this.typeFunction = typeFunction; + this.formatBodyFunction = formatBodyFunction; + this.ifNoneExistFunction = ifNoneExistFunction; + } + + @Setup + public void initClient() throws IOException { + this.client = new HttpHealthcareApiClient(); + } + + @ProcessElement + public void create(ProcessContext context) throws IOException { + T input = context.element(); + String body = formatBodyFunction.apply(input); + String type = typeFunction.apply(input); + try { + // Validate that data was set to valid JSON. + mapper.readTree(input.toString()); + if (ifNoneExistFunction != null) { + String ifNoneExistQuery = ifNoneExistFunction.apply(input); + client.fhirCreate(fhirStore, type, body, ifNoneExistQuery); + } else { + client.fhirCreate(fhirStore, type, body, null); + } + } catch (IOException | HealthcareHttpException e) { + failedBundles.inc(); + context.output(HealthcareIOError.of(body, e)); + } + } + } + } + + /** + * Conditional update conditional update. + * + * @param the type parameter + * @param fhirStore the fhir store + * @return the conditional update + */ + public static ConditionalUpdate conditionalUpdate(ValueProvider fhirStore) { + return new ConditionalUpdate(fhirStore); + } + + /** + * Conditional update conditional update. + * + * @param the type parameter + * @param fhirStore the fhir store + * @return the conditional update + */ + public static ConditionalUpdate conditionalUpdate(String fhirStore) { + return new ConditionalUpdate(fhirStore); + } + + /** + * {@link PTransform} to perform Conditional updates on the FHIR store. + * + *

https://cloud.google.com/healthcare/docs/reference/rest/v1beta1/projects.locations.datasets.fhirStores.fhir/conditionalUpdate + */ + public static class ConditionalUpdate extends PTransform, Write.Result> { + private final ValueProvider fhirStore; + private SerializableFunction> searchParametersFunction; + private SerializableFunction typeFunction; + private SerializableFunction formatBodyFunction; + private SerializableFunction etagFunction; + + /** + * Instantiates a new Conditional update. + * + * @param fhirStore the fhir store + */ + ConditionalUpdate(ValueProvider fhirStore) { + this.fhirStore = fhirStore; + } + + /** + * Instantiates a new Conditional update. + * + * @param fhirStore the fhir store + */ + ConditionalUpdate(String fhirStore) { + this.fhirStore = StaticValueProvider.of(fhirStore); + } + + /** + * With search parameters function conditional update. + * + * @param searchParametersFunction the search parameters function + * @return the conditional update + */ + public ConditionalUpdate withSearchParametersFunction( + SerializableFunction> searchParametersFunction) { + this.searchParametersFunction = searchParametersFunction; + return this; + } + + /** + * With type function conditional update. + * + * @param typeFunction the type function + * @return the conditional update + */ + public ConditionalUpdate withTypeFunction(SerializableFunction typeFunction) { + this.typeFunction = typeFunction; + return this; + } + + /** + * With format body function conditional update. + * + * @param formatBodyFunction the format body function + * @return the conditional update + */ + public ConditionalUpdate withFormatBodyFunction( + SerializableFunction formatBodyFunction) { + this.formatBodyFunction = formatBodyFunction; + return this; + } + + public ConditionalUpdate withEtagFunction(SerializableFunction etagFunction) { + this.etagFunction = etagFunction; + return this; + } + + @Override + public FhirIO.Write.Result expand(PCollection input) { + checkArgument( + typeFunction != null, + "FhirIO.ConditionalUpdate should always be called with a " + "withTypeFunction."); + checkArgument( + formatBodyFunction != null, + "FhirIO.ConditionalUpdate should always be called with a " + "withFormatBodyFunction."); + checkArgument( + searchParametersFunction != null, + "FhirIO.ConditionalUpdate should always be called with a " + + "withSearchParametersFunction. If this does not meet your use case consider usiing " + + "FhirIO.UpdateResources."); + + return Write.Result.in( + input.getPipeline(), + input + .apply( + ParDo.of( + new ConditionalUpdateFn( + fhirStore, + typeFunction, + searchParametersFunction, + formatBodyFunction, + etagFunction))) + .setCoder(HealthcareIOErrorCoder.of(StringUtf8Coder.of()))); + } + + static class ConditionalUpdateFn extends DoFn> { + + private Counter failedConditionalUpdates = + Metrics.counter(ConditionalUpdateFn.class, "failed-conditional-updates"); + private transient HealthcareApiClient client; + private final ObjectMapper mapper = new ObjectMapper(); + private final ValueProvider fhirStore; + private SerializableFunction> searchParametersFunction; + private SerializableFunction typeFunction; + private SerializableFunction formatBodyFunction; + private SerializableFunction etagFunction; + + ConditionalUpdateFn( + ValueProvider fhirStore, + SerializableFunction typeFunction, + SerializableFunction> searchParametersFunction, + SerializableFunction formatBodyFunction, + @Nullable SerializableFunction etagFunction) { + this.fhirStore = fhirStore; + this.typeFunction = typeFunction; + this.searchParametersFunction = searchParametersFunction; + this.formatBodyFunction = formatBodyFunction; + this.etagFunction = etagFunction; + } + + @Setup + public void initClient() throws IOException { + this.client = new HttpHealthcareApiClient(); + } + + @ProcessElement + public void conditionalUpdate(ProcessContext context) throws IOException { + T input = context.element(); + String type = typeFunction.apply(input); + String body = formatBodyFunction.apply(input); + try { + // Validate that data was set to valid JSON. + mapper.readTree(body); + Map searchParameters = searchParametersFunction.apply(input); + String etag = null; + if (etagFunction != null) { + etag = etagFunction.apply(input); + } + client.fhirConditionalUpdate(fhirStore.get(), type, body, searchParameters, etag); + } catch (IOException | HealthcareHttpException e) { + failedConditionalUpdates.inc(); + context.output(HealthcareIOError.of(body, e)); + } + } + } + } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2Message.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2Message.java index 8e846db5b57d..5cf3d2cdbfea 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2Message.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2Message.java @@ -22,8 +22,10 @@ import java.io.IOException; import java.util.Map; import javax.annotation.Nullable; +import org.apache.beam.sdk.coders.DefaultCoder; /** The type HL7v2 message to wrap the {@link Message} model. */ +@DefaultCoder(HL7v2MessageCoder.class) public class HL7v2Message { private static final String schematizedDataKey = "schematizedData"; private static final String schematizedDataPrefix = "{data="; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareApiClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareApiClient.java index f504235a78b5..990e4aa839e7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareApiClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareApiClient.java @@ -27,6 +27,7 @@ import com.google.api.services.healthcare.v1beta1.model.Operation; import java.io.IOException; import java.text.ParseException; +import java.util.Map; import javax.annotation.Nullable; import org.apache.beam.sdk.io.gcp.healthcare.HttpHealthcareApiClient.HealthcareHttpException; import org.joda.time.Instant; @@ -123,7 +124,7 @@ ListMessagesResponse makeHL7v2ListRequest( IngestMessageResponse ingestHL7v2Message(String hl7v2Store, Message msg) throws IOException; /** - * Create hl 7 v 2 message message. + * ConditionalUpdate hl 7 v 2 message message. * * @param hl7v2Store the hl 7 v 2 store * @param msg the msg @@ -149,6 +150,23 @@ Operation pollOperation(Operation operation, Long sleepMs) HttpBody executeFhirBundle(String fhirStore, String bundle) throws IOException, HealthcareHttpException; + HttpBody fhirCreate(String fhirStore, String type, String resource, @Nullable String ifNoneExist) + throws IOException, HealthcareHttpException; + + HttpBody fhirConditionalUpdate( + String fhirStore, + String relativeResourceName, + String resource, + Map searchParameters, + @Nullable String etag) + throws IOException, HealthcareHttpException; + + HttpBody fhirUpdate(String fhirStore, String type, String resource, @Nullable String etag) + throws IOException, HealthcareHttpException; + + HttpBody fhirSearch(String fhirStore, @Nullable String type, Map searchParameters) + throws IOException, HealthcareHttpException; + /** * Read fhir resource http body. * diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java index 653e9a855d68..efd2df5c812f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java @@ -47,6 +47,7 @@ import java.net.URISyntaxException; import java.text.ParseException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -54,6 +55,7 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer; +import org.apache.beam.sdk.io.gcp.healthcare.HttpHealthcareApiClient.FhirHttpRequest.Method; import org.apache.beam.sdk.util.ReleaseInfo; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; import org.apache.http.HttpEntity; @@ -84,9 +86,10 @@ public class HttpHealthcareApiClient implements HealthcareApiClient, Serializabl private static final String FHIRSTORE_HEADER_CONTENT_TYPE = "application/fhir+json"; private static final String FHIRSTORE_HEADER_ACCEPT = "application/fhir+json; charset=utf-8"; private static final String FHIRSTORE_HEADER_ACCEPT_CHARSET = "utf-8"; + private static final String IF_MATCH_HEADER = "If-Match"; private static final Logger LOG = LoggerFactory.getLogger(HttpHealthcareApiClient.class); private transient CloudHealthcare client; - private transient HttpClient httpClient; + public transient HttpClient httpClient; private transient GoogleCredentials credentials; /** @@ -396,35 +399,109 @@ public Operation pollOperation(Operation operation, Long sleepMs) return operation; } - @Override - public HttpBody executeFhirBundle(String fhirStore, String bundle) + static class FhirHttpRequest { + private final String fhirStore; + private final String payload; + private Method method; + private String pathSuffix; + private Map headers; + private Map parameters; + + enum Method { + GET, + POST, + PUT + } + + FhirHttpRequest(String fhirStore, @Nullable String payload) { + this.fhirStore = fhirStore; + this.payload = payload; + this.method = Method.POST; + this.headers = new HashMap<>(); + this.parameters = new HashMap<>(); + } + + public static FhirHttpRequest of(String fhirStore, String payload) { + return new FhirHttpRequest(fhirStore, payload); + } + + public FhirHttpRequest setPathSuffix(String pathSuffix) { + this.pathSuffix = pathSuffix; + return this; + } + + public FhirHttpRequest setHeaders(Map headers) { + this.headers = headers; + return this; + } + + public FhirHttpRequest setMethod(Method method) { + this.method = method; + return this; + } + + public FhirHttpRequest setParameters(Map parameters) { + this.parameters = parameters; + return this; + } + } + + public HttpBody executeFhirHttpRequest(FhirHttpRequest fhirHttpRequest) throws IOException, HealthcareHttpException { if (httpClient == null || client == null) { initClient(); } credentials.refreshIfExpired(); - StringEntity requestEntity = new StringEntity(bundle, ContentType.APPLICATION_JSON); + StringEntity requestEntity = + new StringEntity(fhirHttpRequest.payload, ContentType.APPLICATION_JSON); URI uri; try { - uri = - new URIBuilder(client.getRootUrl() + "v1beta1/" + fhirStore + "/fhir") - .setParameter("access_token", credentials.getAccessToken().getTokenValue()) - .build(); + String baseFhirUri = client.getRootUrl() + "v1beta1/" + fhirHttpRequest.fhirStore + "/fhir"; + String uriString = baseFhirUri; + if (!Strings.isNullOrEmpty(fhirHttpRequest.pathSuffix)) { + uriString += fhirHttpRequest.pathSuffix; + } + uri = new URIBuilder(uriString).build(); } catch (URISyntaxException e) { LOG.error("URL error when making executeBundle request to FHIR API. " + e.getMessage()); throw new IllegalArgumentException(e); } - HttpUriRequest request = - RequestBuilder.post() - .setUri(uri) - .setEntity(requestEntity) - .addHeader("User-Agent", USER_AGENT) - .addHeader("Content-Type", FHIRSTORE_HEADER_CONTENT_TYPE) - .addHeader("Accept-Charset", FHIRSTORE_HEADER_ACCEPT_CHARSET) - .addHeader("Accept", FHIRSTORE_HEADER_ACCEPT) - .build(); + RequestBuilder requestBuilder; + switch (fhirHttpRequest.method) { + case GET: + requestBuilder = RequestBuilder.get(); + break; + case PUT: + requestBuilder = RequestBuilder.put(); + break; + case POST: // fallthrough + default: + requestBuilder = RequestBuilder.post(); + } + + // add common headers + requestBuilder + .setUri(uri) + .setEntity(requestEntity) + .addHeader("Authorization", "Bearer " + credentials.getAccessToken().getTokenValue()) + .addHeader("User-Agent", USER_AGENT) + .addHeader("Content-Type", FHIRSTORE_HEADER_CONTENT_TYPE) + .addHeader("Accept-Charset", FHIRSTORE_HEADER_ACCEPT_CHARSET) + .addHeader("Accept", FHIRSTORE_HEADER_ACCEPT); + + // add additional parameters + for (Map.Entry param : fhirHttpRequest.parameters.entrySet()) { + requestBuilder.addParameter(param.getKey(), param.getValue()); + } + + // add additional parameters + for (Map.Entry param : fhirHttpRequest.headers.entrySet()) { + requestBuilder.addHeader(param.getKey(), param.getValue()); + } + + HttpUriRequest request = requestBuilder.build(); HttpResponse response = httpClient.execute(request); HttpEntity responseEntity = response.getEntity(); @@ -438,6 +515,70 @@ public HttpBody executeFhirBundle(String fhirStore, String bundle) return responseModel; } + @Override + public HttpBody executeFhirBundle(String fhirStore, String bundle) + throws IOException, HealthcareHttpException { + return executeFhirHttpRequest(FhirHttpRequest.of(fhirStore, bundle)); + } + + @Override + public HttpBody fhirCreate( + String fhirStore, String type, String resource, @Nullable String ifNoneExist) + throws IOException, HealthcareHttpException { + Map headers = new HashMap<>(); + if (Strings.isNullOrEmpty(ifNoneExist)) { + headers.put("If-None-Exist", ifNoneExist); + } + return executeFhirHttpRequest( + FhirHttpRequest.of(fhirStore, resource).setPathSuffix("/" + type).setHeaders(headers)); + } + + @Override + public HttpBody fhirConditionalUpdate( + String fhirStore, + String type, + String resource, + Map searchParameters, + @Nullable String etag) + throws IOException, HealthcareHttpException { + Map headers = new HashMap<>(); + if (etag != null) { + headers.put(IF_MATCH_HEADER, etag); + } + return executeFhirHttpRequest( + FhirHttpRequest.of(fhirStore, resource) + .setHeaders(headers) + .setPathSuffix("/" + type) + .setParameters(searchParameters) + .setMethod(Method.PUT)); + } + + @Override + public HttpBody fhirUpdate( + String fhirStore, String relativeResourceName, String resource, @Nullable String etag) + throws IOException, HealthcareHttpException { + Map headers = new HashMap<>(); + if (etag != null) { + headers.put(IF_MATCH_HEADER, etag); + } + return executeFhirHttpRequest( + FhirHttpRequest.of(fhirStore, resource) + .setHeaders(headers) + .setPathSuffix("/" + relativeResourceName) + .setMethod(Method.PUT)); + } + + @Override + public HttpBody fhirSearch( + String fhirStore, @Nullable String type, Map searchParameters) + throws IOException, HealthcareHttpException { + return executeFhirHttpRequest( + new FhirHttpRequest(fhirStore, null) + .setPathSuffix(type) + .setMethod(Method.GET) + .setParameters(searchParameters)); + } + /** * Wraps {@link HttpResponse} in an exception with a statusCode field for use with {@link * HealthcareIOError}. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java index 07f227ad250a..94ad154ff7b0 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java @@ -59,6 +59,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Reshuffle; @@ -81,6 +82,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; @@ -1022,84 +1024,98 @@ public void populateDisplayData(DisplayData.Builder builder) { @Override public SpannerWriteResult expand(PCollection input) { + PCollection> batches; + + if (spec.getBatchSizeBytes() <= 1 + || spec.getMaxNumMutations() <= 1 + || spec.getMaxNumRows() <= 1) { + LOG.info("Batching of mutationGroups is disabled"); + TypeDescriptor> descriptor = + new TypeDescriptor>() {}; + batches = + input.apply(MapElements.into(descriptor).via(element -> ImmutableList.of(element))); + } else { - // First, read the Cloud Spanner schema. - PCollection schemaSeed = - input.getPipeline().apply("Create Seed", Create.of((Void) null)); - if (spec.getSchemaReadySignal() != null) { - // Wait for external signal before reading schema. - schemaSeed = schemaSeed.apply("Wait for schema", Wait.on(spec.getSchemaReadySignal())); + // First, read the Cloud Spanner schema. + PCollection schemaSeed = + input.getPipeline().apply("Create Seed", Create.of((Void) null)); + if (spec.getSchemaReadySignal() != null) { + // Wait for external signal before reading schema. + schemaSeed = schemaSeed.apply("Wait for schema", Wait.on(spec.getSchemaReadySignal())); + } + final PCollectionView schemaView = + schemaSeed + .apply( + "Read information schema", + ParDo.of(new ReadSpannerSchema(spec.getSpannerConfig()))) + .apply("Schema View", View.asSingleton()); + + // Split the mutations into batchable and unbatchable mutations. + // Filter out mutation groups too big to be batched. + PCollectionTuple filteredMutations = + input + .apply( + "RewindowIntoGlobal", + Window.into(new GlobalWindows()) + .triggering(DefaultTrigger.of()) + .discardingFiredPanes()) + .apply( + "Filter Unbatchable Mutations", + ParDo.of( + new BatchableMutationFilterFn( + schemaView, + UNBATCHABLE_MUTATIONS_TAG, + spec.getBatchSizeBytes(), + spec.getMaxNumMutations(), + spec.getMaxNumRows())) + .withSideInputs(schemaView) + .withOutputTags( + BATCHABLE_MUTATIONS_TAG, TupleTagList.of(UNBATCHABLE_MUTATIONS_TAG))); + + // Build a set of Mutation groups from the current bundle, + // sort them by table/key then split into batches. + PCollection> batchedMutations = + filteredMutations + .get(BATCHABLE_MUTATIONS_TAG) + .apply( + "Gather And Sort", + ParDo.of( + new GatherBundleAndSortFn( + spec.getBatchSizeBytes(), + spec.getMaxNumMutations(), + spec.getMaxNumRows(), + // Do not group on streaming unless explicitly set. + spec.getGroupingFactor() + .orElse( + input.isBounded() == IsBounded.BOUNDED + ? DEFAULT_GROUPING_FACTOR + : 1), + schemaView)) + .withSideInputs(schemaView)) + .apply( + "Create Batches", + ParDo.of( + new BatchFn( + spec.getBatchSizeBytes(), + spec.getMaxNumMutations(), + spec.getMaxNumRows(), + schemaView)) + .withSideInputs(schemaView)); + + // Merge the batched and unbatchable mutation PCollections and write to Spanner. + batches = + PCollectionList.of(filteredMutations.get(UNBATCHABLE_MUTATIONS_TAG)) + .and(batchedMutations) + .apply("Merge", Flatten.pCollections()); } - final PCollectionView schemaView = - schemaSeed - .apply( - "Read information schema", - ParDo.of(new ReadSpannerSchema(spec.getSpannerConfig()))) - .apply("Schema View", View.asSingleton()); - - // Split the mutations into batchable and unbatchable mutations. - // Filter out mutation groups too big to be batched. - PCollectionTuple filteredMutations = - input - .apply( - "RewindowIntoGlobal", - Window.into(new GlobalWindows()) - .triggering(DefaultTrigger.of()) - .discardingFiredPanes()) - .apply( - "Filter Unbatchable Mutations", - ParDo.of( - new BatchableMutationFilterFn( - schemaView, - UNBATCHABLE_MUTATIONS_TAG, - spec.getBatchSizeBytes(), - spec.getMaxNumMutations(), - spec.getMaxNumRows())) - .withSideInputs(schemaView) - .withOutputTags( - BATCHABLE_MUTATIONS_TAG, TupleTagList.of(UNBATCHABLE_MUTATIONS_TAG))); - - // Build a set of Mutation groups from the current bundle, - // sort them by table/key then split into batches. - PCollection> batchedMutations = - filteredMutations - .get(BATCHABLE_MUTATIONS_TAG) - .apply( - "Gather And Sort", - ParDo.of( - new GatherBundleAndSortFn( - spec.getBatchSizeBytes(), - spec.getMaxNumMutations(), - spec.getMaxNumRows(), - // Do not group on streaming unless explicitly set. - spec.getGroupingFactor() - .orElse( - input.isBounded() == IsBounded.BOUNDED - ? DEFAULT_GROUPING_FACTOR - : 1), - schemaView)) - .withSideInputs(schemaView)) - .apply( - "Create Batches", - ParDo.of( - new BatchFn( - spec.getBatchSizeBytes(), - spec.getMaxNumMutations(), - spec.getMaxNumRows(), - schemaView)) - .withSideInputs(schemaView)); - - // Merge the batchable and unbatchable mutation PCollections and write to Spanner. + PCollectionTuple result = - PCollectionList.of(filteredMutations.get(UNBATCHABLE_MUTATIONS_TAG)) - .and(batchedMutations) - .apply("Merge", Flatten.pCollections()) - .apply( - "Write mutations to Spanner", - ParDo.of( - new WriteToSpannerFn( - spec.getSpannerConfig(), spec.getFailureMode(), FAILED_MUTATIONS_TAG)) - .withOutputTags(MAIN_OUT_TAG, TupleTagList.of(FAILED_MUTATIONS_TAG))); + batches.apply( + "Write batches to Spanner", + ParDo.of( + new WriteToSpannerFn( + spec.getSpannerConfig(), spec.getFailureMode(), FAILED_MUTATIONS_TAG)) + .withOutputTags(MAIN_OUT_TAG, TupleTagList.of(FAILED_MUTATIONS_TAG))); return new SpannerWriteResult( input.getPipeline(), diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOCreateIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOCreateIT.java new file mode 100644 index 000000000000..68cb6fae2d9f --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOCreateIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.healthcare; + +import static org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.DEFAULT_TEMP_BUCKET; +import static org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.RESOURCES; +import static org.apache.beam.sdk.io.gcp.healthcare.HL7v2IOTestUtil.HEALTHCARE_DATASET_TEMPLATE; + +import java.io.IOException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Collection; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIO.Write.Result; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.ExtractIDSearchParams; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.GetByKey; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class FhirIOCreateIT { + + @Parameters(name = "{0}") + public static Collection versions() { + return Arrays.asList("DSTU2", "STU3", "R4"); + } + + private final String fhirStoreName; + private FhirIOTestOptions options; + private transient HealthcareApiClient client; + private String healthcareDataset; + private long testTime = System.currentTimeMillis(); + + public String version; + + @Rule public transient TestPipeline pipeline = TestPipeline.create(); + + public FhirIOCreateIT(String version) { + this.version = version; + this.fhirStoreName = + "FHIR_store_" + version + "_write_it_" + testTime + "_" + (new SecureRandom().nextInt(32)); + } + + @Before + public void setup() throws Exception { + if (client == null) { + client = new HttpHealthcareApiClient(); + } + PipelineOptionsFactory.register(FhirIOTestOptions.class); + String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + healthcareDataset = String.format(HEALTHCARE_DATASET_TEMPLATE, project); + options = TestPipeline.testingPipelineOptions().as(FhirIOTestOptions.class); + options.setGcsTempPath( + String.format("gs://%s/FhirIOWrite%sIT/%s/temp/", DEFAULT_TEMP_BUCKET, version, testTime)); + options.setGcsDeadLetterPath( + String.format( + "gs://%s/FhirIOWrite%sIT/%s/deadletter/", DEFAULT_TEMP_BUCKET, version, testTime)); + options.setFhirStore(healthcareDataset + "/fhirStores/" + fhirStoreName); + HealthcareApiClient client = new HttpHealthcareApiClient(); + client.createFhirStore(healthcareDataset, fhirStoreName, version); + } + + @After + public void teardownFhirStore() throws IOException { + HealthcareApiClient client = new HttpHealthcareApiClient(); + client.deleteFhirStore(healthcareDataset + "/fhirStores/" + fhirStoreName); + // clean up GCS objects if any. + } + + @AfterClass + public static void teardownBucket() throws IOException { + FhirIOTestUtil.tearDownTempBucket(); + } + + @Test + public void testFhirIO_CreateResources() { + Result createResult = + (Result) + pipeline + .apply("Seed Test Resources", Create.of(RESOURCES.get(version))) + .apply( + "Create FHIR Resources", + FhirIO.createResources(options.getFhirStore()) + .withTypeFunction(new GetByKey("resourceType")) + .withIfNotExistFunction(new ExtractIDSearchParams()) + .withFormatBodyFunction(x -> x)); + + PAssert.that(createResult.getFailedBodies()).empty(); + + pipeline.run().waitUntilFinish(); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java index c893f59f744b..b96d22bef1d0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.healthcare; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.javanet.NetHttpTransport; @@ -27,6 +28,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -39,13 +41,81 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.beam.sdk.io.gcp.healthcare.HttpHealthcareApiClient.HealthcareHttpException; +import org.apache.beam.sdk.transforms.SerializableFunction; class FhirIOTestUtil { public static final String DEFAULT_TEMP_BUCKET = "temp-storage-for-healthcare-io-tests"; - private static Stream readPrettyBundles(String version) { - ClassLoader classLoader = FhirIOTestUtil.class.getClassLoader(); - Path resourceDir = Paths.get("build", "resources", "test", version); + public static class ExtractIDSearchQuery implements SerializableFunction { + private ObjectMapper mapper; + + ExtractIDSearchQuery() { + mapper = new ObjectMapper(); + } + + @Override + public String apply(String resource) { + try { + Map map = + mapper.readValue(resource.getBytes(StandardCharsets.UTF_8), Map.class); + String id = map.get("id"); + return String.format("_id=%s", id); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static class ExtractIDSearchParams + implements SerializableFunction> { + private ObjectMapper mapper; + + ExtractIDSearchParams() { + mapper = new ObjectMapper(); + } + + @Override + public Map apply(String resource) { + Map searchParams = new HashMap<>(); + try { + Map map = + mapper.readValue(resource.getBytes(StandardCharsets.UTF_8), Map.class); + String id = map.get("id"); + searchParams.put("_id", id); + return searchParams; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static class GetByKey implements SerializableFunction { + private final String key; + private ObjectMapper mapper; + + public GetByKey(String key) { + this.key = key; + mapper = new ObjectMapper(); + } + + @Override + public String apply(String resource) { + try { + Map map = + mapper.readValue(resource.getBytes(StandardCharsets.UTF_8), Map.class); + return map.get(key); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + // TODO read initial resources function. + // TODO read update resources function. + // TODO spot check resource update utility. + + private static Stream readAllTestResources(String subDir, String version) { + Path resourceDir = Paths.get("build", "resources", "test", subDir, version); String absolutePath = resourceDir.toFile().getAbsolutePath(); File dir = new File(absolutePath); File[] fhirJsons = dir.listFiles(); @@ -62,14 +132,21 @@ private static Stream readPrettyBundles(String version) { .map(String::new); } + private static List readPrettyBundles(String version) { + return readAllTestResources("transactional_bundles", version).collect(Collectors.toList()); + } + + private static List readResources(String version) { + return readAllTestResources("resources", version) // stream of file contents + .map((String x) -> x.split("\\r?\\n")) // split lines + .flatMap(Arrays::stream) // flatten lines for all files + .collect(Collectors.toList()); + } // Could generate more messages at scale using a tool like // https://synthetichealth.github.io/synthea/ if necessary chose not to avoid the dependency. - static final List DSTU2_PRETTY_BUNDLES = - readPrettyBundles("DSTU2").collect(Collectors.toList()); - static final List STU3_PRETTY_BUNDLES = - readPrettyBundles("STU3").collect(Collectors.toList()); - static final List R4_PRETTY_BUNDLES = - readPrettyBundles("R4").collect(Collectors.toList()); + static final List DSTU2_PRETTY_BUNDLES = readPrettyBundles("DSTU2"); + static final List STU3_PRETTY_BUNDLES = readPrettyBundles("STU3"); + static final List R4_PRETTY_BUNDLES = readPrettyBundles("R4"); static final Map> BUNDLES; @@ -81,6 +158,22 @@ private static Stream readPrettyBundles(String version) { BUNDLES = Collections.unmodifiableMap(m); } + // Could generate more messages at scale using a tool like + // https://synthetichealth.github.io/synthea/ if necessary chose not to avoid the dependency. + static final List DSTU2_RESOURCES = readResources("DSTU2"); + static final List STU3_RESOURCES = readResources("STU3"); + static final List R4_RESOURCES = readResources("R4"); + + static final Map> RESOURCES; + + static { + Map> m = new HashMap<>(); + m.put("DSTU2", DSTU2_RESOURCES); + m.put("STU3", STU3_RESOURCES); + m.put("R4", R4_RESOURCES); + RESOURCES = Collections.unmodifiableMap(m); + } + /** Populate the test resources into the FHIR store and returns a list of resource IDs. */ static void executeFhirBundles(HealthcareApiClient client, String fhirStore, List bundles) throws IOException, HealthcareHttpException { @@ -89,6 +182,18 @@ static void executeFhirBundles(HealthcareApiClient client, String fhirStore, Lis } } + /** Populate the test resources into the FHIR store and returns a list of resource IDs. */ + static void createFhirResources( + HealthcareApiClient client, String fhirStore, List resources) + throws IOException, HealthcareHttpException { + GetByKey getByKey = new GetByKey("resourceType"); + ExtractIDSearchQuery extractIDSearchQuery = new ExtractIDSearchQuery(); + for (String resource : resources) { + client.fhirCreate( + fhirStore, getByKey.apply(resource), resource, extractIDSearchQuery.apply(resource)); + } + } + public static void tearDownTempBucket() throws IOException { GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOUpdateIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOUpdateIT.java new file mode 100644 index 000000000000..8e095cc0802e --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOUpdateIT.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.healthcare; + +import static org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.DEFAULT_TEMP_BUCKET; +import static org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.RESOURCES; +import static org.apache.beam.sdk.io.gcp.healthcare.HL7v2IOTestUtil.HEALTHCARE_DATASET_TEMPLATE; + +import java.io.IOException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Collection; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIO.Write.Result; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.ExtractIDSearchParams; +import org.apache.beam.sdk.io.gcp.healthcare.FhirIOTestUtil.GetByKey; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class FhirIOUpdateIT { + + @Parameters(name = "{0}") + public static Collection versions() { + // TODO(jaketf) uncomment other versions. + return Arrays.asList(/*"DSTU2", "STU3", */ "R4"); + } + + private final String fhirStoreName; + private FhirIOTestOptions options; + private transient HealthcareApiClient client; + private String healthcareDataset; + private long testTime = System.currentTimeMillis(); + + public String version; + + @Rule public transient TestPipeline pipeline = TestPipeline.create(); + + public FhirIOUpdateIT(String version) { + this.version = version; + this.fhirStoreName = + "FHIR_store_" + version + "_write_it_" + testTime + "_" + (new SecureRandom().nextInt(32)); + } + + @Before + public void setup() throws Exception { + if (client == null) { + client = new HttpHealthcareApiClient(); + } + PipelineOptionsFactory.register(FhirIOTestOptions.class); + String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + healthcareDataset = String.format(HEALTHCARE_DATASET_TEMPLATE, project); + options = TestPipeline.testingPipelineOptions().as(FhirIOTestOptions.class); + options.setGcsTempPath( + String.format("gs://%s/FhirIOWrite%sIT/%s/temp/", DEFAULT_TEMP_BUCKET, version, testTime)); + options.setGcsDeadLetterPath( + String.format( + "gs://%s/FhirIOWrite%sIT/%s/deadletter/", DEFAULT_TEMP_BUCKET, version, testTime)); + options.setFhirStore(healthcareDataset + "/fhirStores/" + fhirStoreName); + HealthcareApiClient client = new HttpHealthcareApiClient(); + client.createFhirStore(healthcareDataset, fhirStoreName, version); + FhirIOTestUtil.createFhirResources( + client, healthcareDataset + "/fhirStores/" + fhirStoreName, RESOURCES.get(version)); + } + + @After + public void teardownFhirStore() throws IOException { + HealthcareApiClient client = new HttpHealthcareApiClient(); + client.deleteFhirStore(healthcareDataset + "/fhirStores/" + fhirStoreName); + // clean up GCS objects if any. + } + + @AfterClass + public static void teardownBucket() throws IOException { + FhirIOTestUtil.tearDownTempBucket(); + } + + @Test + public void testFhirIO_ConditionalUpdate() { + Result conditionalUpdateResult = + (Result) + pipeline + .apply("Create Test Resources", Create.of(RESOURCES.get(version))) + .apply( + "Conditional Update Resources", + FhirIO.conditionalUpdate(options.getFhirStore()) + .withTypeFunction(new GetByKey("resourceType")) + .withFormatBodyFunction(x -> x) + .withSearchParametersFunction(new ExtractIDSearchParams())); + pipeline.run().waitUntilFinish(); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadIT.java index d3769b8fa3db..c0e981fd88b1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadIT.java @@ -73,7 +73,7 @@ public void setup() throws Exception { if (client == null) { this.client = new HttpHealthcareApiClient(); } - // Create HL7 messages and write them to HL7v2 Store. + // CreateResources HL7 messages and write them to HL7v2 Store. writeHL7v2Messages(this.client, healthcareDataset + "/hl7V2Stores/" + HL7V2_STORE_NAME); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadWriteIT.java index 8a4566bffc26..395b5563cb44 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IOReadWriteIT.java @@ -81,7 +81,7 @@ public void setup() throws Exception { client = new HttpHealthcareApiClient(); } - // Create HL7 messages and write them to HL7v2 Store. + // CreateResources HL7 messages and write them to HL7v2 Store. writeHL7v2Messages(this.client, healthcareDataset + "/hl7V2Stores/" + INPUT_HL7V2_STORE_NAME); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java index ffcbaefff9a5..816556cf1bc3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java @@ -263,18 +263,34 @@ private void verifyBatches(Iterable... batches) { @Test public void noBatching() throws Exception { + + // This test uses a different mock/fake because it explicitly does not want to populate the + // Spanner schema. + FakeServiceFactory fakeServiceFactory = new FakeServiceFactory(); + ReadOnlyTransaction tx = mock(ReadOnlyTransaction.class); + when(fakeServiceFactory.mockDatabaseClient().readOnlyTransaction()).thenReturn(tx); + + // Capture batches sent to writeAtLeastOnce. + when(fakeServiceFactory.mockDatabaseClient().writeAtLeastOnce(mutationBatchesCaptor.capture())) + .thenReturn(null); + PCollection mutations = pipeline.apply(Create.of(g(m(1L)), g(m(2L)))); mutations.apply( SpannerIO.write() .withProjectId("test-project") .withInstanceId("test-instance") .withDatabaseId("test-database") - .withServiceFactory(serviceFactory) + .withServiceFactory(fakeServiceFactory) .withBatchSizeBytes(1) .grouped()); pipeline.run(); - verifyBatches(batch(m(1L)), batch(m(2L))); + verify(fakeServiceFactory.mockDatabaseClient(), times(1)) + .writeAtLeastOnce(mutationsInNoOrder(batch(m(1L)))); + verify(fakeServiceFactory.mockDatabaseClient(), times(1)) + .writeAtLeastOnce(mutationsInNoOrder(batch(m(2L)))); + // If no batching then the DB schema is never read. + verify(tx, never()).executeQuery(any()); } @Test diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/CarePlan.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/CarePlan.ndjson new file mode 100644 index 000000000000..a4034d143b89 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/CarePlan.ndjson @@ -0,0 +1,3 @@ +{"resourceType":"CarePlan","id":"ba15e4c6-414b-4385-8abf-7be2eae77983","text":{"status":"generated","div":"

Respiratory therapy
"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","context":{"reference":"Encounter/429ca9b0-f562-4845-aff6-0b6fc3aa1085"},"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-19T14:15:05-07:00"},"category":[{"coding":[{"system":"http://snomed.info/sct","code":"53950000","display":"Respiratory therapy"}],"text":"Respiratory therapy"}],"addresses":[{"reference":"Condition/46d525ba-6571-44de-a6b0-e43a3810bc8e"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"304510005","display":"Recommendation to avoid exercise"}],"text":"Recommendation to avoid exercise"},"status":"completed","prohibited":false}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"371605008","display":"Deep breathing and coughing exercises"}],"text":"Deep breathing and coughing exercises"},"status":"completed","prohibited":false}}]} +{"resourceType":"CarePlan","id":"c9f29319-842a-4528-8866-b915f558d66b","text":{"status":"generated","div":"
Fracture care
"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","context":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"},"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-12-29T13:15:05-08:00"},"category":[{"coding":[{"system":"http://snomed.info/sct","code":"385691007","display":"Fracture care"}],"text":"Fracture care"}],"addresses":[{"reference":"Condition/b1cecfd7-b8b9-4159-98c8-95e7ad63d74a"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed","prohibited":false}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"408580007","display":"Physical activity target light exercise"}],"text":"Physical activity target light exercise"},"status":"completed","prohibited":false}}]} +{"resourceType":"CarePlan","id":"0b324605-d5b0-4356-a80b-59148979151a","text":{"status":"generated","div":"
Minor surgery care management (procedure)
"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","context":{"reference":"Encounter/dd792a18-dcd6-4d70-ad52-78a04ab63018"},"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-23T13:15:05-08:00"},"category":[{"coding":[{"system":"http://snomed.info/sct","code":"737471002","display":"Minor surgery care management (procedure)"}],"text":"Minor surgery care management (procedure)"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed","prohibited":false}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"243077000","display":"Recommendation to limit sexual activity"}],"text":"Recommendation to limit sexual activity"},"status":"completed","prohibited":false}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Claim.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Claim.ndjson new file mode 100644 index 000000000000..49cc9d98c2b9 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Claim.ndjson @@ -0,0 +1,25 @@ +{"resourceType":"Claim","id":"fd5133df-a283-4ed2-a0a1-600c5f781504","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/759887f5-cdb3-42aa-ba72-403b3c1017ec"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"40f42d0e-2644-4b93-b81e-aa8333f5d107","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","diagnosis":[{"sequence":1,"diagnosis":{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}}]} +{"resourceType":"Claim","id":"c3d7130b-eab8-472f-86d6-21dd373ad796","type":"institutional","organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"},"net":{"value":504.83,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"f0059888-64fd-4177-b90f-dbb450d95914","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/e1582c1a-e002-40be-8e5e-041d71b32945"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"aea74302-dc46-4032-80b9-dfc41748edf3","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}}]} +{"resourceType":"Claim","id":"8e1db8cc-ce4c-426c-8421-a9d518a30a73","type":"institutional","organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"a53e8d15-f6ab-4c5a-b984-c60796304d9c","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/442bf25b-fd87-4bec-9c71-7fff3389f54d"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"7fab04c8-ea33-4cc3-a050-0b8008a50409","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}}]} +{"resourceType":"Claim","id":"98f46e66-f451-47db-81cc-f77bed1e183c","type":"institutional","organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":4,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":5,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"75510ac1-f51c-4fad-a255-711d8d1acfbe","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/2339a360-d66a-4518-848b-f2ebdf0db27e"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"89f97d6a-9511-4cf8-84fe-9c8369af8188","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}}]} +{"resourceType":"Claim","id":"ea6a35dc-0be4-4be9-90ec-88f58800b35b","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/417abd1f-43e3-446f-8b4e-3b237bc6c1ba"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"966e64bb-385a-4ac0-b97e-4cfa225ecea5","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","diagnosis":[{"sequence":1,"diagnosis":{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"},"net":{"value":6535.58,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"f731e722-2f3b-4c90-8488-2595ae8374b6","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"0b196692-83fa-4edd-beab-fc1a4e3a31b3","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/fe1a0af0-3bb4-4864-a7f6-8bc91a6f29e6"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"e1b59b3b-65ee-45f0-ae4d-5c69fadd3e94","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}}]} +{"resourceType":"Claim","id":"a975e6b7-b0ee-4df0-b1f3-406da196f899","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/8fcc3aa1-2288-4c60-9670-8c756295d7e1"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"37aa3b3c-0bd6-4756-bdba-a19c17b6f3ff","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/7dc37b2f-c065-48fc-a5e4-c0973fe705b7"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"2cd90bd3-cfd7-4ddc-a7dc-e5df0443f070","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","diagnosis":[{"sequence":1,"diagnosis":{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":4,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"02e90c71-8d21-4e91-8e0a-d948446ebddc","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}}]} +{"resourceType":"Claim","id":"24df9742-07df-44f8-a09d-823fee928063","type":"institutional","organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"},"net":{"value":482.02,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"617f658b-6244-4e9b-b53e-3edaa636b540","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}}]} +{"resourceType":"Claim","id":"8371259c-ea6c-458f-b7d1-4e18c36e976d","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}},{"sequence":2,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"},"net":{"value":9364.49,"system":"urn:iso:std:iso:4217","code":"USD"}}]} +{"resourceType":"Claim","id":"c135ee04-a1be-4b2e-9b47-1b0cb7939d6a","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","prescription":{"reference":"MedicationOrder/19a955e5-961c-4042-95fa-3d3ddefdd75d"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"}} +{"resourceType":"Claim","id":"4490461e-44df-42c1-ab4d-83bb8e081632","type":"institutional","organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"use":"complete","diagnosis":[{"sequence":1,"diagnosis":{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"item":[{"sequence":1,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}},{"sequence":3,"type":{"system":"http://hl7.org/fhir/v3/ActCode","code":"CSINV","display":"clinical service invoice"},"service":{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"},"net":{"value":1958.61,"system":"urn:iso:std:iso:4217","code":"USD"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Condition.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Condition.ndjson new file mode 100644 index 000000000000..03293ca003c5 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Condition.ndjson @@ -0,0 +1,4 @@ +{"resourceType":"Condition","id":"6468ef65-32eb-43c0-a292-498217bfe77d","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c596d869-aee8-47be-be1f-9044a0b03056"},"dateRecorded":"2010-05-19","code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"category":{"coding":[{"system":"http://hl7.org/fhir/condition-category","code":"diagnosis"}]},"clinicalStatus":"resolved","verificationStatus":"confirmed","onsetDateTime":"2010-05-19T14:15:05-07:00","abatementDateTime":"2010-05-26T14:15:05-07:00"} +{"resourceType":"Condition","id":"46d525ba-6571-44de-a6b0-e43a3810bc8e","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/429ca9b0-f562-4845-aff6-0b6fc3aa1085"},"dateRecorded":"2015-04-12","code":{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"},"category":{"coding":[{"system":"http://hl7.org/fhir/condition-category","code":"diagnosis"}]},"clinicalStatus":"resolved","verificationStatus":"confirmed","onsetDateTime":"2015-04-12T14:15:05-07:00","abatementDateTime":"2015-04-19T14:15:05-07:00"} +{"resourceType":"Condition","id":"b1cecfd7-b8b9-4159-98c8-95e7ad63d74a","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"},"dateRecorded":"2016-10-30","code":{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"},"category":{"coding":[{"system":"http://hl7.org/fhir/condition-category","code":"diagnosis"}]},"clinicalStatus":"resolved","verificationStatus":"confirmed","onsetDateTime":"2016-10-30T14:15:05-07:00","abatementDateTime":"2016-12-29T13:15:05-08:00"} +{"resourceType":"Condition","id":"83443d35-4ad0-43f9-9d04-a38fd1a40546","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/bfb92e78-2a6d-4a28-8cac-9511f1ec98ce"},"dateRecorded":"2018-12-11","code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"category":{"coding":[{"system":"http://hl7.org/fhir/condition-category","code":"diagnosis"}]},"clinicalStatus":"resolved","verificationStatus":"confirmed","onsetDateTime":"2018-12-11T13:15:05-08:00","abatementDateTime":"2018-12-19T13:15:05-08:00"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/DiagnosticReport.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/DiagnosticReport.ndjson new file mode 100644 index 000000000000..446b6c943cfe --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/DiagnosticReport.ndjson @@ -0,0 +1 @@ +{"resourceType":"DiagnosticReport","id":"202599a0-96e1-40d7-ad50-7759082e833b","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/ValueSet/diagnostic-service-sections","code":"LAB"}]},"code":{"coding":[{"system":"http://loinc.org","code":"58410-2","display":"Complete blood count (hemogram) panel - Blood by Automated count"}],"text":"Complete blood count (hemogram) panel - Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","performer":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"result":[{"reference":"Observation/af682f69-afe1-4d95-b2ff-701e99b48057","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Encounter.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Encounter.ndjson new file mode 100644 index 000000000000..53f484883471 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Encounter.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"Encounter","id":"c596d869-aee8-47be-be1f-9044a0b03056","status":"finished","class":"ambulatory","type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"2a0b33f5-4f48-468a-b814-16415492c421","status":"finished","type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"f1965c82-d450-4740-956c-1d5481c2bc3f","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"f85534f8-57c0-4c87-ba1a-9c51240b2749","status":"finished","type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"269a52f1-6e55-474b-b6f7-ad4c4acf4929","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"c416f8eb-c3b5-4c45-bfb8-4617e9050442","status":"finished","type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"ed094139-44fd-483c-bf6f-38fdbb1e2925","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"429ca9b0-f562-4845-aff6-0b6fc3aa1085","status":"finished","class":"ambulatory","type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"76264224-5098-4cef-8731-f96423f99704","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}],"text":"Encounter for check up (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"3e7926ca-2d44-4f15-a7a7-2879e7103e7a","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"0059d564-708d-467b-b8d8-449492ad93c7","status":"finished","class":"emergency","type":[{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"402eba83-dc9f-4be5-bf3f-a4c074d730e2","status":"finished","class":"ambulatory","type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}],"text":"Encounter for 'check-up'"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c","status":"finished","type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"3c725b99-002f-468c-accc-5037dc124942","status":"finished","class":"outpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"dd792a18-dcd6-4d70-ad52-78a04ab63018","status":"finished","class":"inpatient","type":[{"coding":[{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}],"text":"Admission to surgical department"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"bfb92e78-2a6d-4a28-8cac-9511f1ec98ce","status":"finished","class":"ambulatory","type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/ImagingStudy.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/ImagingStudy.ndjson new file mode 100644 index 000000000000..a59b691dd652 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/ImagingStudy.ndjson @@ -0,0 +1 @@ +{"resourceType":"ImagingStudy","id":"b981d431-fd72-4325-878f-1bcf0e3bc321","started":"2016-10-30T14:15:05-07:00","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"uid":"urn:oid:1.2.840.99999999.22502730.1589831196459","numberOfSeries":1,"numberOfInstances":1,"series":[{"number":1,"modality":{"system":"http://dicom.nema.org/resources/ontology/DCM","code":"DX","display":"Digital Radiography"},"uid":"urn:oid:1.2.840.99999999.1.65971777.1589831196459","numberOfInstances":1,"availability":"UNAVAILABLE","bodySite":{"system":"http://snomed.info/sct","code":"51299004","display":"Clavicle"},"started":"2016-10-30T14:15:05-07:00","instance":[{"number":1,"uid":"urn:oid:1.2.840.99999999.1.1.48010377.1589831196459","sopClass":"urn:oid:1.2.840.10008.5.1.4.1.1.1.1","title":"Image of clavicle"}]}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Immunization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Immunization.ndjson new file mode 100644 index 000000000000..fa03bd45f190 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Immunization.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"Immunization","id":"217e6566-fef3-4cfc-9bc1-b4dda86f4a7d","status":"completed","date":"2010-06-03T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"}} +{"resourceType":"Immunization","id":"0a185e37-eed6-4196-baeb-edb7d2334923","status":"completed","date":"2011-06-09T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"}} +{"resourceType":"Immunization","id":"388eac61-1d07-4787-bdac-8cdf0a7426a9","status":"completed","date":"2014-06-12T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"}} +{"resourceType":"Immunization","id":"92394c6c-5eb5-4169-ac5f-78dc240a0347","status":"completed","date":"2014-06-12T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"}],"text":"Td (adult) preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"}} +{"resourceType":"Immunization","id":"d9d6410c-a65e-4b6c-a2f2-1d55ac41a1b5","status":"completed","date":"2014-06-12T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"}} +{"resourceType":"Immunization","id":"37c95cd2-6fb8-4aad-9303-716f6cee7a60","status":"completed","date":"2014-06-12T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"}],"text":"meningococcal MCV4P"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"}} +{"resourceType":"Immunization","id":"b8be2715-bd14-4627-adbf-70d0b27a2c9e","status":"completed","date":"2015-04-23T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"}} +{"resourceType":"Immunization","id":"f7a65385-4028-470a-a365-ee9836782642","status":"completed","date":"2015-04-23T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"}} +{"resourceType":"Immunization","id":"e93c7361-bfd5-459b-ad13-de4b0bbfaa45","status":"completed","date":"2017-06-15T14:15:05-07:00","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"wasNotGiven":false,"reported":false,"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/MedicationOrder.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/MedicationOrder.ndjson new file mode 100644 index 000000000000..a29d53277cff --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/MedicationOrder.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"MedicationOrder","id":"759887f5-cdb3-42aa-ba72-403b3c1017ec","dateWritten":"2010-05-19T14:15:05-07:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/c596d869-aee8-47be-be1f-9044a0b03056"},"reasonReference":{"reference":"Condition/6468ef65-32eb-43c0-a292-498217bfe77d"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"}} +{"resourceType":"MedicationOrder","id":"e1582c1a-e002-40be-8e5e-041d71b32945","dateWritten":"2011-03-11T13:15:05-08:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/f1965c82-d450-4740-956c-1d5481c2bc3f"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"751905","display":"Trinessa 28 Day Pack"}],"text":"Trinessa 28 Day Pack"}} +{"resourceType":"MedicationOrder","id":"442bf25b-fd87-4bec-9c71-7fff3389f54d","dateWritten":"2013-02-28T13:15:05-08:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/269a52f1-6e55-474b-b6f7-ad4c4acf4929"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"748856","display":"Yaz 28 Day Pack"}],"text":"Yaz 28 Day Pack"}} +{"resourceType":"MedicationOrder","id":"2339a360-d66a-4518-848b-f2ebdf0db27e","dateWritten":"2015-02-18T13:15:05-08:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/ed094139-44fd-483c-bf6f-38fdbb1e2925"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"1534809","display":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"}],"text":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"}} +{"resourceType":"MedicationOrder","id":"417abd1f-43e3-446f-8b4e-3b237bc6c1ba","dateWritten":"2015-04-12T14:15:05-07:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/429ca9b0-f562-4845-aff6-0b6fc3aa1085"},"reasonReference":{"reference":"Condition/46d525ba-6571-44de-a6b0-e43a3810bc8e"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"313782","display":"Acetaminophen 325 MG Oral Tablet"}],"text":"Acetaminophen 325 MG Oral Tablet"}} +{"resourceType":"MedicationOrder","id":"fe1a0af0-3bb4-4864-a7f6-8bc91a6f29e6","dateWritten":"2016-02-13T13:15:05-08:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/3e7926ca-2d44-4f15-a7a7-2879e7103e7a"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"978950","display":"Natazia 28 Day Pack"}],"text":"Natazia 28 Day Pack"}} +{"resourceType":"MedicationOrder","id":"8fcc3aa1-2288-4c60-9670-8c756295d7e1","dateWritten":"2016-10-30T14:15:05-07:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"861467","display":"Meperidine Hydrochloride 50 MG Oral Tablet"}],"text":"Meperidine Hydrochloride 50 MG Oral Tablet"},"dosageInstruction":[{"timing":{"repeat":{"frequency":1,"period":4.0,"periodUnits":"h"}},"asNeededBoolean":false,"doseQuantity":{"value":1.0}}]} +{"resourceType":"MedicationOrder","id":"7dc37b2f-c065-48fc-a5e4-c0973fe705b7","dateWritten":"2016-10-30T14:15:05-07:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"310965","display":"Ibuprofen 200 MG Oral Tablet"}],"text":"Ibuprofen 200 MG Oral Tablet"},"dosageInstruction":[{"asNeededBoolean":true}]} +{"resourceType":"MedicationOrder","id":"19a955e5-961c-4042-95fa-3d3ddefdd75d","dateWritten":"2018-12-11T13:15:05-08:00","status":"stopped","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"prescriber":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"encounter":{"reference":"Encounter/bfb92e78-2a6d-4a28-8cac-9511f1ec98ce"},"reasonReference":{"reference":"Condition/83443d35-4ad0-43f9-9d04-a38fd1a40546"},"medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Observation.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Observation.ndjson new file mode 100644 index 000000000000..a139a7c21cc7 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Observation.ndjson @@ -0,0 +1,54 @@ +{"resourceType":"Observation","id":"ace52964-c945-4434-8bce-7dc8334ebb1f","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":153.6,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"280ac889-62a2-4f40-b1df-3ee1bf55c44a","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":1,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"5c69fc59-93ae-4569-881a-90c95e713783","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":55.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"da3b05a9-7c2e-47bb-99a6-c292e7e3410a","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":23.41,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"b3ff3362-0faa-4d82-adbb-c2e9abf66b26","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":71.788,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"fa401b31-c5c0-4773-9f2f-9e2ac2faf530","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":72,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":120,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"5dd09901-1878-4c33-8bea-901867ab8ef6","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"14727409-d62b-448d-8739-f7a34fbda73d","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"1049a0e2-da6e-4972-a1bb-59255b4f0915","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}],"text":"survey"},"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"6c52e341-718e-48a6-9619-91ec7fe15a71","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":153.7,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"e3e7db52-6dfe-4f72-9970-387f6ddd1103","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"f315729d-f344-4fb0-a67c-e2acdb128bb3","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":62.1,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"4d63298f-9a97-44e1-814c-5d82dbb662d5","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":26.3,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"f4823f99-c40c-4a5b-84da-0c581f6b0510","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":85.491,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"0a550125-9dae-4acb-a4c6-5464fe580dbe","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":88,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":121,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"19aced8c-1ed9-4dbe-b829-9499fe1e8324","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":93,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"65b982c9-1a37-40ba-a459-06f57d0e8272","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"b4e25ecd-57bc-4f7a-b73c-038292bbc60a","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}],"text":"survey"},"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f85534f8-57c0-4c87-ba1a-9c51240b2749"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"7bd7f85e-ddca-4d45-b813-7d92169da64e","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"d93dc239-a2bf-4a59-8379-26ccb39cfe67","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"4402d329-81a3-41ae-9179-3b23a4896f26","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":69.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"fefb4870-44be-4f80-8f1e-c1ef963e22d1","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":29.25,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"01a3e5b1-f2cf-4aa7-85d0-9c99610e9e7f","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":87,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":122,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"0a1c6fb9-1614-4e7e-9c8c-5aaf3da2897f","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":90,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"8a7a8549-1260-4e5f-bfaa-ba27f27b44ca","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":16,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"6a7cd9eb-829d-4771-8109-b2234a7b0c49","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"6690-2","display":"Leukocytes [#/volume] in Blood by Automated count"}],"text":"Leukocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":7.2857,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"097a47bf-7ae9-471f-a081-87b165c81f63","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"789-8","display":"Erythrocytes [#/volume] in Blood by Automated count"}],"text":"Erythrocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":4.7109,"unit":"10*6/uL","system":"http://unitsofmeasure.org","code":"10*6/uL"}} +{"resourceType":"Observation","id":"6c6590f8-3fbc-4cdd-93fd-d40ab785abbf","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"718-7","display":"Hemoglobin [Mass/volume] in Blood"}],"text":"Hemoglobin [Mass/volume] in Blood"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":15.964,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"4c1c503e-70b7-4d4e-8751-762c16a202a2","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"4544-3","display":"Hematocrit [Volume Fraction] of Blood by Automated count"}],"text":"Hematocrit [Volume Fraction] of Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":45.934,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"1b89993a-e7ae-4620-a10c-7aeb598a75a9","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"787-2","display":"MCV [Entitic volume] by Automated count"}],"text":"MCV [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":94.793,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"78a08462-e203-4ec1-a9ce-51b29262080b","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"785-6","display":"MCH [Entitic mass] by Automated count"}],"text":"MCH [Entitic mass] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":28.937,"unit":"pg","system":"http://unitsofmeasure.org","code":"pg"}} +{"resourceType":"Observation","id":"ba5f4798-0ea9-451e-a943-99d048d05c0b","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"786-4","display":"MCHC [Mass/volume] by Automated count"}],"text":"MCHC [Mass/volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":33.479,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"621b2d18-f34d-446d-b35e-dfefc0acb826","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"21000-5","display":"Erythrocyte distribution width [Entitic volume] by Automated count"}],"text":"Erythrocyte distribution width [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":43.303,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"512bdee3-dc80-41dc-8d15-c220a7504ebf","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"777-3","display":"Platelets [#/volume] in Blood by Automated count"}],"text":"Platelets [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":173.89,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"1ab93bb4-1867-432f-b5d7-002a43b314a1","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"32207-3","display":"Platelet distribution width [Entitic volume] in Blood by Automated count"}],"text":"Platelet distribution width [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":413.95,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"af682f69-afe1-4d95-b2ff-701e99b48057","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}],"text":"laboratory"},"code":{"coding":[{"system":"http://loinc.org","code":"32623-1","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}],"text":"Platelet mean volume [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":10.07,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"1c2bfa7f-80e8-430e-922b-6d0d51f409d6","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}],"text":"survey"},"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/c416f8eb-c3b5-4c45-bfb8-4617e9050442"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"3fd81e92-32c5-4516-b66b-2bb3abe1ae6c","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"ce4b4ec2-8b75-4270-8822-52e801961e88","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"d47d450a-a881-4aa7-8281-37059113722f","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":70.3,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"c78bef2c-6c68-4039-b1d9-1dbdd18b648b","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":29.73,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"4c6b48d2-8b38-47e6-868f-a7e38888a471","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":79,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":129,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"2737d9c5-60b3-471a-a5d0-f5e2b5763485","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":90,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"23762ec7-dba6-4114-96d1-0b8f91d28194","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":15,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"fb274daa-b9a8-423f-b10a-9f74ec1bf383","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}],"text":"survey"},"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/76264224-5098-4cef-8731-f96423f99704"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"2b7d5771-4912-4901-ba2d-ed4956f549cc","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"74f833a2-5b10-4482-a8f7-7b672b54fdfd","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":0,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"2958f9d7-227b-476b-8d6f-0d96dea1b2fd","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66.8,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"bb5ffa55-1a1d-4b8b-8ccc-39bcbc38af58","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":28.22,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"fda5887c-49ba-4494-b263-bd6c5124f3b8","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":69,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":111,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"0853005c-9c07-4da4-b91a-22e4fb4145b7","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"d960be5d-3b83-4a5a-8f8b-653c9fdd79b7","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":12,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"131cd0ab-612d-4b9e-97b8-db72573d684d","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}],"text":"survey"},"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"fe4d9e94-f3ff-48b0-a731-4f7fd1a68ad3","status":"final","category":{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}],"text":"vital-signs"},"code":{"coding":[{"system":"http://loinc.org","code":"8310-5","display":"Body temperature"}],"text":"Body temperature"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/bfb92e78-2a6d-4a28-8cac-9511f1ec98ce"},"effectiveDateTime":"2018-12-11T13:15:05-08:00","issued":"2018-12-11T13:15:05.225-08:00","valueQuantity":{"value":39.024,"unit":"Cel","system":"http://unitsofmeasure.org","code":"Cel"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Organization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Organization.ndjson new file mode 100644 index 000000000000..6e8753228fa1 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Organization.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Organization","id":"465de31f-3098-365c-af70-48a071e1f5aa","identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"465de31f-3098-365c-af70-48a071e1f5aa"}],"type":{"coding":[{"system":"Healthcare Provider","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"},"name":"METROWEST MEDICAL CENTER","telecom":[{"system":"phone","value":"5083831000"}],"address":[{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}]} +{"resourceType":"Organization","id":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}],"type":{"coding":[{"system":"Healthcare Provider","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"},"name":"PCP68975","telecom":[{"system":"phone","value":"508-881-4368"}],"address":[{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Patient.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Patient.ndjson new file mode 100644 index 000000000000..d8d4f4a9c389 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Patient.ndjson @@ -0,0 +1 @@ +{"resourceType":"Patient","id":"1416dec1-f4b1-4b48-b7f4-650e8f67499c","text":{"status":"generated","div":"
Generated by Synthea.Version identifier: v2.5.0-265-gbd5a00e8\n . Person seed: 6732543839779682504 Population seed: 1589831189867
"},"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/us-core-race","valueCodeableConcept":{"coding":[{"system":"http://hl7.org/fhir/v3/Race","code":"2106-3","display":"White"}],"text":"White"}},{"url":"http://hl7.org/fhir/StructureDefinition/us-core-ethnicity","valueCodeableConcept":{"coding":[{"system":"http://hl7.org/fhir/v3/Ethnicity","code":"2186-5","display":"Not Hispanic or Latino"}],"text":"Not Hispanic or Latino"}},{"url":"http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName","valueString":"Leanna255 Predovic534"},{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex","valueCode":"F"},{"url":"http://hl7.org/fhir/StructureDefinition/birthPlace","valueAddress":{"city":"Southbridge","state":"Massachusetts","country":"US"}},{"url":"http://synthetichealth.github.io/synthea/disability-adjusted-life-years","valueDecimal":0.0},{"url":"http://synthetichealth.github.io/synthea/quality-adjusted-life-years","valueDecimal":27.0}],"identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://hl7.org/fhir/v2/0203","code":"MR"}]},"system":"http://hospital.smarthealthit.org","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://hl7.org/fhir/identifier-type","code":"SB"}]},"system":"http://hl7.org/fhir/sid/us-ssn","value":"999-54-3579"},{"type":{"coding":[{"system":"http://hl7.org/fhir/v2/0203","code":"DL"}]},"system":"urn:oid:2.16.840.1.113883.4.3.25","value":"S99972984"}],"name":[{"use":"official","family":["Nolan344"],"given":["Lorita217"],"prefix":["Ms."]}],"telecom":[{"system":"phone","value":"555-817-6998","use":"home"}],"gender":"female","birthDate":"1992-04-09","address":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/geolocation","extension":[{"url":"latitude","valueDecimal":42.27693107900605},{"url":"longitude","valueDecimal":-71.45741653702677}]}],"line":["330 Sawayn Parade"],"city":"Framingham","state":"Massachusetts","country":"US"}],"maritalStatus":{"coding":[{"system":"http://hl7.org/fhir/v3/MaritalStatus","code":"S"}]},"multipleBirthBoolean":false,"communication":[{"language":{"coding":[{"system":"urn:ietf:bcp:47","code":"en-US","display":"English"}],"text":"English"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Practitioner.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Practitioner.ndjson new file mode 100644 index 000000000000..e40c5c9d1f46 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Practitioner.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Practitioner","id":"c16820ae-2954-32d4-863c-e9ceb741154c","identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"530"}],"active":true,"name":{"family":["Murphy561"],"given":["Mari763"],"prefix":["Dr."]},"address":[{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}],"gender":"female"} +{"resourceType":"Practitioner","id":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"35750"}],"active":true,"name":{"family":["Hilpert278"],"given":["Cathryn51"],"prefix":["Dr."]},"address":[{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}],"gender":"female"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Procedure.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Procedure.ndjson new file mode 100644 index 000000000000..c86efacdeb1b --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/DSTU2/Procedure.ndjson @@ -0,0 +1,7 @@ +{"resourceType":"Procedure","id":"c5141e34-d584-4900-b59c-ef099efe22dc","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"performedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:30:05-07:00"},"encounter":{"reference":"Encounter/2a0b33f5-4f48-468a-b814-16415492c421"}} +{"resourceType":"Procedure","id":"2349f028-54e4-43fd-8060-2ce6f810924b","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"}],"text":"Plain chest X-ray (procedure)"},"reasonReference":{"reference":"Condition/46d525ba-6571-44de-a6b0-e43a3810bc8e"},"performedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:26:05-07:00"},"encounter":{"reference":"Encounter/429ca9b0-f562-4845-aff6-0b6fc3aa1085"}} +{"resourceType":"Procedure","id":"ef872ffd-e080-4432-bd24-1d41aef41ed7","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"}],"text":"Clavicle X-ray"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T14:45:05-07:00"},"encounter":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"}} +{"resourceType":"Procedure","id":"61a3a3f6-bcad-49ef-ab79-ea87fab516cc","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"}],"text":"Admission to orthopedic department"},"reasonReference":{"reference":"Condition/b1cecfd7-b8b9-4159-98c8-95e7ad63d74a"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T15:15:05-07:00"},"encounter":{"reference":"Encounter/0059d564-708d-467b-b8d8-449492ad93c7"}} +{"resourceType":"Procedure","id":"cdfebd64-8e72-4aec-bd6a-2be27eb41b82","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"performedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:30:05-07:00"},"encounter":{"reference":"Encounter/6ca65d01-35b1-4b58-8ee7-d4a6cedd2f5c"}} +{"resourceType":"Procedure","id":"b34c7fd4-6bb2-4c62-9d3c-98aa4e125251","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"}],"text":"Bilateral tubal ligation"},"performedPeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-09T15:15:05-08:00"},"encounter":{"reference":"Encounter/dd792a18-dcd6-4d70-ad52-78a04ab63018"}} +{"resourceType":"Procedure","id":"edf461c8-c51f-4161-bc66-7b61e12627af","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"}],"text":"Throat culture (procedure)"},"reasonReference":{"reference":"Condition/83443d35-4ad0-43f9-9d04-a38fd1a40546"},"performedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:30:05-08:00"},"encounter":{"reference":"Encounter/bfb92e78-2a6d-4a28-8cac-9511f1ec98ce"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CarePlan.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CarePlan.ndjson new file mode 100644 index 000000000000..12ba93cc62a8 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CarePlan.ndjson @@ -0,0 +1,3 @@ +{"resourceType":"CarePlan","id":"32841402-d5d8-4615-af90-473bf6f417fa","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careplan"]},"text":{"status":"generated","div":"
Care Plan for Respiratory therapy.
Activities:
  • Respiratory therapy
  • Respiratory therapy

Care plan is meant to treat Acute bronchitis (disorder).
"},"status":"completed","intent":"order","category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/careplan-category","code":"assess-plan"}]},{"coding":[{"system":"http://snomed.info/sct","code":"53950000","display":"Respiratory therapy"}],"text":"Respiratory therapy"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-19T14:15:05-07:00"},"careTeam":[{"reference":"CareTeam/111c8459-549e-4a2b-ab70-27f3b2408eac"}],"addresses":[{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"304510005","display":"Recommendation to avoid exercise"}],"text":"Recommendation to avoid exercise"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"371605008","display":"Deep breathing and coughing exercises"}],"text":"Deep breathing and coughing exercises"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}}]} +{"resourceType":"CarePlan","id":"ac2893f1-9c90-437d-852f-4cc1a0155d39","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careplan"]},"text":{"status":"generated","div":"
Care Plan for Fracture care.
Activities:
  • Fracture care
  • Fracture care

Care plan is meant to treat Fracture of clavicle.
"},"status":"completed","intent":"order","category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/careplan-category","code":"assess-plan"}]},{"coding":[{"system":"http://snomed.info/sct","code":"385691007","display":"Fracture care"}],"text":"Fracture care"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-12-29T13:15:05-08:00"},"careTeam":[{"reference":"CareTeam/31d5cf9a-c745-4087-8042-6a9b9babaef9"}],"addresses":[{"reference":"Condition/115098be-c26e-47d0-9cad-be9df75d7042"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"408580007","display":"Physical activity target light exercise"}],"text":"Physical activity target light exercise"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}}]} +{"resourceType":"CarePlan","id":"5a1fb9d1-310b-4fe3-b7c3-6370529400b8","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careplan"]},"text":{"status":"generated","div":"
Care Plan for Minor surgery care management (procedure).
Activities:
  • Minor surgery care management (procedure)
  • Minor surgery care management (procedure)
"},"status":"completed","intent":"order","category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/careplan-category","code":"assess-plan"}]},{"coding":[{"system":"http://snomed.info/sct","code":"737471002","display":"Minor surgery care management (procedure)"}],"text":"Minor surgery care management (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"},"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-23T13:15:05-08:00"},"careTeam":[{"reference":"CareTeam/e1ab345a-81bd-461e-921d-9eae42c47f8a"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"243077000","display":"Recommendation to limit sexual activity"}],"text":"Recommendation to limit sexual activity"},"status":"completed","location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CareTeam.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CareTeam.ndjson new file mode 100644 index 000000000000..1bd6a0f44212 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/CareTeam.ndjson @@ -0,0 +1,3 @@ +{"resourceType":"CareTeam","id":"111c8459-549e-4a2b-ab70-27f3b2408eac","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careteam"]},"status":"inactive","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-19T14:15:05-07:00"},"participant":[{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"116153009","display":"Patient"}],"text":"Patient"}],"member":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Person in the healthcare environment (person)"}],"text":"Person in the healthcare environment (person)"}],"member":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Healthcare related organization (qualifier value)"}],"text":"Healthcare related organization (qualifier value)"}],"member":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}}],"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"}],"managingOrganization":[{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}]} +{"resourceType":"CareTeam","id":"31d5cf9a-c745-4087-8042-6a9b9babaef9","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careteam"]},"status":"inactive","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-12-29T13:15:05-08:00"},"participant":[{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"116153009","display":"Patient"}],"text":"Patient"}],"member":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Person in the healthcare environment (person)"}],"text":"Person in the healthcare environment (person)"}],"member":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Healthcare related organization (qualifier value)"}],"text":"Healthcare related organization (qualifier value)"}],"member":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}}],"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"}],"managingOrganization":[{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}]} +{"resourceType":"CareTeam","id":"e1ab345a-81bd-461e-921d-9eae42c47f8a","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-careteam"]},"status":"inactive","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"},"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-23T13:15:05-08:00"},"participant":[{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"116153009","display":"Patient"}],"text":"Patient"}],"member":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Person in the healthcare environment (person)"}],"text":"Person in the healthcare environment (person)"}],"member":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}},{"role":[{"coding":[{"system":"http://snomed.info/sct","code":"303118004","display":"Healthcare related organization (qualifier value)"}],"text":"Healthcare related organization (qualifier value)"}],"member":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}}],"managingOrganization":[{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Claim.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Claim.ndjson new file mode 100644 index 000000000000..05a7d0cf7222 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Claim.ndjson @@ -0,0 +1,25 @@ +{"resourceType":"Claim","id":"e24412c9-d798-4a45-8d29-98027d1eaa5d","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"created":"2010-05-19T14:45:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/8a21a17d-b0b3-48b2-8427-5eb9d1138628"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"}]}],"total":{"value":10.45,"currency":"USD"}} +{"resourceType":"Claim","id":"a7b49375-f326-462c-91a6-fb68c3d7c359","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"created":"2010-05-19T14:45:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"}]},{"sequence":2,"diagnosisSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"d4f07a36-012e-4170-8757-0c9bfbdfa575","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"created":"2010-06-03T14:45:05-07:00","provider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"supportingInfo":[{"sequence":1,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/f28a9afd-a143-45db-8aeb-082b0df22426"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/eba3e367-30ea-418d-ae97-e91506dea85f"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"encounter":[{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"}]},{"sequence":2,"informationSequence":[1],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"currency":"USD"}},{"sequence":3,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"net":{"value":504.83,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"e4ae0e04-c0b5-47c0-9f1c-1a4cefe48a97","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"created":"2011-03-11T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/c9954767-c8b7-45e0-8b07-099351dcd9a0"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"}]}],"total":{"value":31.0,"currency":"USD"}} +{"resourceType":"Claim","id":"6556aaa0-bf9f-40e7-bc54-1c3ceec565c7","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"created":"2011-03-11T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"d8f548d1-bf09-4fb3-a0d0-5454feee2396","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"created":"2011-06-09T14:30:05-07:00","provider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"supportingInfo":[{"sequence":1,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/e755857d-8ce3-42b6-81f0-d5b52524fae5"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"encounter":[{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"}]},{"sequence":2,"informationSequence":[1],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"2d2de9d8-9f05-4979-b19f-62d0733cbe9b","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"created":"2013-02-28T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/d10f9ae0-ffef-4a9b-8021-74a6cd6b1ad5"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"}]}],"total":{"value":26.21,"currency":"USD"}} +{"resourceType":"Claim","id":"171190a5-7d0d-45a4-99ac-0c519b6dd270","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"created":"2013-02-28T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"50387d60-9d96-475d-9719-7b9cb30071ca","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"created":"2014-06-12T14:30:05-07:00","provider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"supportingInfo":[{"sequence":1,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/490a18a8-30e4-4a65-9a8c-0993671cf186"}},{"sequence":2,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/e451e36e-dc92-4fd9-a924-1a83a04da939"}},{"sequence":3,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/673cd32a-bf63-4c7a-9e57-1b92d9fb15e9"}},{"sequence":4,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/41daff0e-fb2b-4216-8065-1481d7c962ed"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"encounter":[{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"}]},{"sequence":2,"informationSequence":[1],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"currency":"USD"}},{"sequence":3,"informationSequence":[2],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"}],"text":"Td (adult) preservative free"},"net":{"value":140.52,"currency":"USD"}},{"sequence":4,"informationSequence":[3],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"net":{"value":140.52,"currency":"USD"}},{"sequence":5,"informationSequence":[4],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"}],"text":"meningococcal MCV4P"},"net":{"value":140.52,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"39917e96-c35b-4c72-9abf-222dacb4d19e","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"created":"2015-02-18T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/c316061f-f9ad-4a30-80cf-c488b64b1112"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"}]}],"total":{"value":45.32,"currency":"USD"}} +{"resourceType":"Claim","id":"43604c79-af2f-4ea3-a871-a6efeb148686","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"created":"2015-02-18T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"9e1c8ca0-e622-46c2-ae92-5b32f9ebfff0","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"created":"2015-04-12T14:41:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/4b2bf1a7-6801-4ed0-85d2-336106458fdc"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"}]}],"total":{"value":5.32,"currency":"USD"}} +{"resourceType":"Claim","id":"5c4943b7-52fe-43cf-b800-200585136db8","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"created":"2015-04-12T14:41:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/78d69a34-6c80-4a6c-abbb-0accca0a17a5"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"}]},{"sequence":2,"diagnosisSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"}},{"sequence":3,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"}],"text":"Plain chest X-ray (procedure)"},"net":{"value":6535.58,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"99fc6fc4-98f4-415a-a094-f22df454163f","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"created":"2015-04-23T14:30:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"supportingInfo":[{"sequence":1,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/aebc3fce-9411-4246-87ee-17a65f3f5f9a"}},{"sequence":2,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/ab327672-36ba-4980-83a8-c4a1525de444"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}],"text":"Encounter for check up (procedure)"},"encounter":[{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"}]},{"sequence":2,"informationSequence":[1],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"currency":"USD"}},{"sequence":3,"informationSequence":[2],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"net":{"value":140.52,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"8c0e3992-24e7-4c44-9c50-3f0ea1431726","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"created":"2016-02-13T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/1df5e2a6-88ef-423c-ae94-5972e6ed7d36"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"}]}],"total":{"value":31.25,"currency":"USD"}} +{"resourceType":"Claim","id":"4a41154a-06aa-4bfc-94ce-b96f28f11b80","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"created":"2016-02-13T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"80484d29-eded-431d-bf9b-8a917089a77e","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"created":"2016-10-30T16:45:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/7202acd4-409c-4078-87bb-125f9ffe4bad"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"},"encounter":[{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"}]}],"total":{"value":113.81,"currency":"USD"}} +{"resourceType":"Claim","id":"41a00b32-22ff-4fd3-a388-296e4cc77dfe","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"created":"2016-10-30T16:45:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/edfc50e7-5843-4cbe-b4d2-b8a0c5b6bcfe"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"},"encounter":[{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"}]}],"total":{"value":17.35,"currency":"USD"}} +{"resourceType":"Claim","id":"07ae31e4-8480-4293-b506-5e9b32e7491b","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"created":"2016-10-30T16:45:05-07:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/115098be-c26e-47d0-9cad-be9df75d7042"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/e614b129-913f-42ac-bcc3-266db50af307"}},{"sequence":2,"procedureReference":{"reference":"Procedure/cf1cfcd7-ca27-4bf1-b8f0-14e31c69824e"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"},"encounter":[{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"}]},{"sequence":2,"diagnosisSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"}},{"sequence":3,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"}],"text":"Clavicle X-ray"},"net":{"value":516.65,"currency":"USD"}},{"sequence":4,"procedureSequence":[2],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"}],"text":"Admission to orthopedic department"},"net":{"value":516.65,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"b60311e1-94d9-473c-bf5f-ea06db1dad15","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"created":"2016-12-29T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}],"text":"Encounter for 'check-up'"},"encounter":[{"reference":"Encounter/a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"b52c9dfa-56bb-4523-9ace-38ad614ee107","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"created":"2017-06-15T14:45:05-07:00","provider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"supportingInfo":[{"sequence":1,"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/bc61721c-4053-4a81-b1e5-b64194e1efa1"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/ea19efb7-6346-45e9-aa8b-3195cc0b8aee"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"encounter":[{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"}]},{"sequence":2,"informationSequence":[1],"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"net":{"value":140.52,"currency":"USD"}},{"sequence":3,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"net":{"value":482.02,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"084d3b6a-e1d4-4e25-b90b-c970fbc20eb7","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"created":"2018-02-02T13:30:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"encounter":[{"reference":"Encounter/eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"}]}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"cf98de5a-9843-43c6-84c6-e6e703ffa457","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"created":"2018-02-10T15:15:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/91112614-fd2f-46cb-ad49-96111084ef1c"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}],"text":"Admission to surgical department"},"encounter":[{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"}]},{"sequence":2,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"}],"text":"Bilateral tubal ligation"},"net":{"value":9364.49,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} +{"resourceType":"Claim","id":"bfc5accc-20be-4556-b76a-e076fc4b2278","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"pharmacy"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"created":"2018-12-11T13:45:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"prescription":{"reference":"MedicationRequest/86aaf462-e2c8-4cdd-8db1-5edb78f15ea1"},"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"}]}],"total":{"value":16.98,"currency":"USD"}} +{"resourceType":"Claim","id":"15be99a3-ce23-43fd-a32a-fbda483a1224","status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Lorita217 Nolan344"},"billablePeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"created":"2018-12-11T13:45:05-08:00","provider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"priority":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/processpriority","code":"normal"}]},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/9c0af3aa-42c9-43a6-8677-ae5fa3f34922"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/1b7f052f-0bb7-422e-86c9-0c8f6af49fe2"}}],"insurance":[{"sequence":1,"focal":true,"coverage":{"display":"Medicaid"}}],"item":[{"sequence":1,"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"encounter":[{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"}]},{"sequence":2,"diagnosisSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"}},{"sequence":3,"procedureSequence":[1],"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"}],"text":"Throat culture (procedure)"},"net":{"value":1958.61,"currency":"USD"}}],"total":{"value":129.16,"currency":"USD"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Condition.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Condition.ndjson new file mode 100644 index 000000000000..138f07decf90 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Condition.ndjson @@ -0,0 +1,4 @@ +{"resourceType":"Condition","id":"4336ae40-60ee-4148-9271-a372cb9600a7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]},"clinicalStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-clinical","code":"resolved"}]},"verificationStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-ver-status","code":"confirmed"}]},"category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-category","code":"encounter-diagnosis","display":"Encounter Diagnosis"}]}],"code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"},"onsetDateTime":"2010-05-19T14:15:05-07:00","abatementDateTime":"2010-05-26T14:15:05-07:00","recordedDate":"2010-05-19T14:15:05-07:00"} +{"resourceType":"Condition","id":"c8d97153-17c7-4a2b-bafd-052ff8f30eb9","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]},"clinicalStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-clinical","code":"resolved"}]},"verificationStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-ver-status","code":"confirmed"}]},"category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-category","code":"encounter-diagnosis","display":"Encounter Diagnosis"}]}],"code":{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"onsetDateTime":"2015-04-12T14:15:05-07:00","abatementDateTime":"2015-04-19T14:15:05-07:00","recordedDate":"2015-04-12T14:15:05-07:00"} +{"resourceType":"Condition","id":"115098be-c26e-47d0-9cad-be9df75d7042","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]},"clinicalStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-clinical","code":"resolved"}]},"verificationStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-ver-status","code":"confirmed"}]},"category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-category","code":"encounter-diagnosis","display":"Encounter Diagnosis"}]}],"code":{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"onsetDateTime":"2016-10-30T14:15:05-07:00","abatementDateTime":"2016-12-29T13:15:05-08:00","recordedDate":"2016-10-30T14:15:05-07:00"} +{"resourceType":"Condition","id":"9c0af3aa-42c9-43a6-8677-ae5fa3f34922","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]},"clinicalStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-clinical","code":"resolved"}]},"verificationStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-ver-status","code":"confirmed"}]},"category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/condition-category","code":"encounter-diagnosis","display":"Encounter Diagnosis"}]}],"code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},"onsetDateTime":"2018-12-11T13:15:05-08:00","abatementDateTime":"2018-12-19T13:15:05-08:00","recordedDate":"2018-12-11T13:15:05-08:00"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DiagnosticReport.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DiagnosticReport.ndjson new file mode 100644 index 000000000000..7d7861955132 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DiagnosticReport.ndjson @@ -0,0 +1,17 @@ +{"resourceType":"DiagnosticReport","id":"c31327f8-f2ec-4b26-a8bd-40aff65cc085","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"},"effectiveDateTime":"2010-05-19T14:15:05-07:00","issued":"2010-05-19T14:15:05.225-07:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTAtMDUtMTkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCk5vIEFjdGl2ZSBNZWRpY2F0aW9ucy4KCiMgQXNzZXNzbWVudCBhbmQgUGxhbgpQYXRpZW50IGlzIHByZXNlbnRpbmcgd2l0aCBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuIAoKIyMgUGxhbgoKVGhlIHBhdGllbnQgd2FzIHByZXNjcmliZWQgdGhlIGZvbGxvd2luZyBtZWRpY2F0aW9uczoKLSBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldAo="}]} +{"resourceType":"DiagnosticReport","id":"08e52e51-ee85-4e52-a192-27d6d817334b","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTAtMDYtMDMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUuIApUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gbWVkaWNhdGlvbiByZWNvbmNpbGlhdGlvbiAocHJvY2VkdXJlKQo="}]} +{"resourceType":"DiagnosticReport","id":"e00fc1e7-2625-4515-8cdf-9b22d41ae0e6","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"},"effectiveDateTime":"2011-03-11T13:15:05-08:00","issued":"2011-03-11T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTEtMDMtMTEKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIHRyaW5lc3NhIDI4IGRheSBwYWNrCg=="}]} +{"resourceType":"DiagnosticReport","id":"c76085ce-ced6-4fb2-8a88-1c62638db794","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTEtMDYtMDkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE5IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjawoKIyBBc3Nlc3NtZW50IGFuZCBQbGFuCgoKIyMgUGxhbgpQYXRpZW50IHdhcyBnaXZlbiB0aGUgZm9sbG93aW5nIGltbXVuaXphdGlvbnM6IGluZmx1ZW56YSwgc2Vhc29uYWwsIGluamVjdGFibGUsIHByZXNlcnZhdGl2ZSBmcmVlLiAK"}]} +{"resourceType":"DiagnosticReport","id":"124c584e-a411-4ef2-8a28-875b40bdffaf","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"},"effectiveDateTime":"2013-02-28T13:15:05-08:00","issued":"2013-02-28T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTMtMDItMjgKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIwIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjawoKIyBBc3Nlc3NtZW50IGFuZCBQbGFuCgoKIyMgUGxhbgoKVGhlIHBhdGllbnQgd2FzIHByZXNjcmliZWQgdGhlIGZvbGxvd2luZyBtZWRpY2F0aW9uczoKLSB5YXogMjggZGF5IHBhY2sK"}]} +{"resourceType":"DiagnosticReport","id":"a5b95e67-0013-4c75-812a-4c1464642fc7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0074","code":"LAB","display":"Laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"58410-2","display":"Complete blood count (hemogram) panel - Blood by Automated count"}],"text":"Complete blood count (hemogram) panel - Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","result":[{"reference":"Observation/4b4e7514-1472-4455-b30d-9632803a4094","display":"Leukocytes [#/volume] in Blood by Automated count"},{"reference":"Observation/53a4daa2-d54e-4ae0-bb11-5a808163f42c","display":"Erythrocytes [#/volume] in Blood by Automated count"},{"reference":"Observation/a6a99aca-ef13-449d-9354-e009d08e8c71","display":"Hemoglobin [Mass/volume] in Blood"},{"reference":"Observation/d60aaa5a-2632-494e-ba83-794ad85127d5","display":"Hematocrit [Volume Fraction] of Blood by Automated count"},{"reference":"Observation/c25c4744-6c81-4a1f-91f8-0f68ddf1f4af","display":"MCV [Entitic volume] by Automated count"},{"reference":"Observation/ff785a7c-4966-4df7-a878-0dcb6de76abb","display":"MCH [Entitic mass] by Automated count"},{"reference":"Observation/c8b433eb-f658-44ae-b7d7-fa0d62cb701a","display":"MCHC [Mass/volume] by Automated count"},{"reference":"Observation/69bcf0d7-7b07-4458-9ff6-788dc2443246","display":"Erythrocyte distribution width [Entitic volume] by Automated count"},{"reference":"Observation/fc5546f2-b5a9-44db-ab27-f04194526e5b","display":"Platelets [#/volume] in Blood by Automated count"},{"reference":"Observation/90029698-691a-430e-bca8-6f54338193fc","display":"Platelet distribution width [Entitic volume] in Blood by Automated count"},{"reference":"Observation/48c316cf-6407-4574-b884-142bb11b0b42","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}]} +{"resourceType":"DiagnosticReport","id":"6c7aed2a-2f9f-4796-85a9-ffa36af2560b","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTQtMDYtMTIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIyIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUsIHRkIChhZHVsdCkgcHJlc2VydmF0aXZlIGZyZWUsIGhlcCBiLCBhZHVsdCwgbWVuaW5nb2NvY2NhbCBtY3Y0cC4gCg=="}]} +{"resourceType":"DiagnosticReport","id":"728336a3-9e9a-4737-8767-b8de22d14385","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"},"effectiveDateTime":"2015-02-18T13:15:05-08:00","issued":"2015-02-18T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTUtMDItMTgKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIyIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW0K"}]} +{"resourceType":"DiagnosticReport","id":"0dfbba83-7fa9-48ab-ab34-a545a161d6e8","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"effectiveDateTime":"2015-04-12T14:15:05-07:00","issued":"2015-04-12T14:15:05.225-07:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTUtMDQtMTIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCjE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggYWN1dGUgYnJvbmNoaXRpcyAoZGlzb3JkZXIpLiAKCiMjIFBsYW4KClRoZSBmb2xsb3dpbmcgcHJvY2VkdXJlcyB3ZXJlIGNvbmR1Y3RlZDoKLSBwbGFpbiBjaGVzdCB4LXJheSAocHJvY2VkdXJlKQpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIGFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0ClRoZSBwYXRpZW50IHdhcyBwbGFjZWQgb24gYSBjYXJlcGxhbjoKLSByZXNwaXJhdG9yeSB0aGVyYXB5Cg=="}]} +{"resourceType":"DiagnosticReport","id":"a359262a-494b-48d5-9eca-52debab2c309","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTUtMDQtMjMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUsIGhlcCBiLCBhZHVsdC4gCg=="}]} +{"resourceType":"DiagnosticReport","id":"ef3508ff-eb22-4144-a6b5-dd745129c055","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"},"effectiveDateTime":"2016-02-13T13:15:05-08:00","issued":"2016-02-13T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTYtMDItMTMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIG5hdGF6aWEgMjggZGF5IHBhY2sK"}]} +{"resourceType":"DiagnosticReport","id":"1a30352d-3766-47dc-b9e6-9dd65f702ab1","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"effectiveDateTime":"2016-10-30T14:15:05-07:00","issued":"2016-10-30T14:15:05.225-07:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTYtMTAtMzAKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI0IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IG5hdGF6aWEgMjggZGF5IHBhY2s7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggZnJhY3R1cmUgb2YgY2xhdmljbGUuIAoKIyMgUGxhbgoKVGhlIGZvbGxvd2luZyBwcm9jZWR1cmVzIHdlcmUgY29uZHVjdGVkOgotIGNsYXZpY2xlIHgtcmF5Ci0gYWRtaXNzaW9uIHRvIG9ydGhvcGVkaWMgZGVwYXJ0bWVudApUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIG1lcGVyaWRpbmUgaHlkcm9jaGxvcmlkZSA1MCBtZyBvcmFsIHRhYmxldAotIGlidXByb2ZlbiAyMDAgbWcgb3JhbCB0YWJsZXQKVGhlIHBhdGllbnQgd2FzIHBsYWNlZCBvbiBhIGNhcmVwbGFuOgotIGZyYWN0dXJlIGNhcmUK"}]} +{"resourceType":"DiagnosticReport","id":"0996a8ad-3dae-4574-97b8-e63bfbed0436","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"},"effectiveDateTime":"2016-12-29T13:15:05-08:00","issued":"2016-12-29T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTYtMTItMjkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI0IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgo="}]} +{"resourceType":"DiagnosticReport","id":"dada411f-d95a-42b6-bea5-987b2da3f0a1","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTctMDYtMTUKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUuIApUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gbWVkaWNhdGlvbiByZWNvbmNpbGlhdGlvbiAocHJvY2VkdXJlKQo="}]} +{"resourceType":"DiagnosticReport","id":"a6d08a29-a694-482d-8df8-36b63c8e8296","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"},"effectiveDateTime":"2018-02-02T13:15:05-08:00","issued":"2018-02-02T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTgtMDItMDIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgo="}]} +{"resourceType":"DiagnosticReport","id":"e1385d55-31f2-4a8b-8356-22869d42d0bd","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"},"effectiveDateTime":"2018-02-09T13:15:05-08:00","issued":"2018-02-09T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTgtMDItMDkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gYmlsYXRlcmFsIHR1YmFsIGxpZ2F0aW9uClRoZSBwYXRpZW50IHdhcyBwbGFjZWQgb24gYSBjYXJlcGxhbjoKLSBtaW5vciBzdXJnZXJ5IGNhcmUgbWFuYWdlbWVudCAocHJvY2VkdXJlKQo="}]} +{"resourceType":"DiagnosticReport","id":"6d6cb707-6e0f-4fe3-a047-8b0b0da44807","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note"]},"status":"final","category":[{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},"effectiveDateTime":"2018-12-11T13:15:05-08:00","issued":"2018-12-11T13:15:05.225-08:00","performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"presentedForm":[{"contentType":"text/plain","data":"CjIwMTgtMTItMTEKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI2IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggc3RyZXB0b2NvY2NhbCBzb3JlIHRocm9hdCAoZGlzb3JkZXIpLiAKCiMjIFBsYW4KClRoZSBmb2xsb3dpbmcgcHJvY2VkdXJlcyB3ZXJlIGNvbmR1Y3RlZDoKLSB0aHJvYXQgY3VsdHVyZSAocHJvY2VkdXJlKQpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0Cg=="}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DocumentReference.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DocumentReference.ndjson new file mode 100644 index 000000000000..c6da76c2a773 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/DocumentReference.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"DocumentReference","id":"a6e558a3-7907-4185-9537-5b3291e05639","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2010-05-19T14:15:05.225-07:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTAtMDUtMTkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCk5vIEFjdGl2ZSBNZWRpY2F0aW9ucy4KCiMgQXNzZXNzbWVudCBhbmQgUGxhbgpQYXRpZW50IGlzIHByZXNlbnRpbmcgd2l0aCBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuIAoKIyMgUGxhbgoKVGhlIHBhdGllbnQgd2FzIHByZXNjcmliZWQgdGhlIGZvbGxvd2luZyBtZWRpY2F0aW9uczoKLSBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldAo="}}],"context":{"encounter":[{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"}],"period":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"}}} +{"resourceType":"DocumentReference","id":"9e6df167-efc0-4b0a-922a-11afba4fe703","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2010-06-03T14:15:05.225-07:00","author":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTAtMDYtMDMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUuIApUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gbWVkaWNhdGlvbiByZWNvbmNpbGlhdGlvbiAocHJvY2VkdXJlKQo="}}],"context":{"encounter":[{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"}],"period":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"}}} +{"resourceType":"DocumentReference","id":"a2a936d2-11dc-46b2-81d2-4f85ba5784ed","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2011-03-11T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTEtMDMtMTEKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE4IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5CiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIHRyaW5lc3NhIDI4IGRheSBwYWNrCg=="}}],"context":{"encounter":[{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"}],"period":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"246b40b8-e424-4be1-b296-de4e7c10e52d","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2011-06-09T14:15:05.225-07:00","author":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTEtMDYtMDkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDE5IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjawoKIyBBc3Nlc3NtZW50IGFuZCBQbGFuCgoKIyMgUGxhbgpQYXRpZW50IHdhcyBnaXZlbiB0aGUgZm9sbG93aW5nIGltbXVuaXphdGlvbnM6IGluZmx1ZW56YSwgc2Vhc29uYWwsIGluamVjdGFibGUsIHByZXNlcnZhdGl2ZSBmcmVlLiAK"}}],"context":{"encounter":[{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"}],"period":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"}}} +{"resourceType":"DocumentReference","id":"99076549-16bc-4843-b6b2-1a912336a5a9","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2013-02-28T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTMtMDItMjgKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIwIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjawoKIyBBc3Nlc3NtZW50IGFuZCBQbGFuCgoKIyMgUGxhbgoKVGhlIHBhdGllbnQgd2FzIHByZXNjcmliZWQgdGhlIGZvbGxvd2luZyBtZWRpY2F0aW9uczoKLSB5YXogMjggZGF5IHBhY2sK"}}],"context":{"encounter":[{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"}],"period":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"f9bba222-4d5f-4b1b-b681-c4ed60c05f89","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2014-06-12T14:15:05.225-07:00","author":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTQtMDYtMTIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIyIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUsIHRkIChhZHVsdCkgcHJlc2VydmF0aXZlIGZyZWUsIGhlcCBiLCBhZHVsdCwgbWVuaW5nb2NvY2NhbCBtY3Y0cC4gCg=="}}],"context":{"encounter":[{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"}],"period":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"}}} +{"resourceType":"DocumentReference","id":"56a84077-f6ac-462b-8c81-40ec2d97b9f6","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2015-02-18T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTUtMDItMTgKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIyIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCnBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW0K"}}],"context":{"encounter":[{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"}],"period":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"f1489031-2951-45c0-8e24-975e7d3eae5d","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2015-04-12T14:15:05.225-07:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTUtMDQtMTIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlcikuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCjE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggYWN1dGUgYnJvbmNoaXRpcyAoZGlzb3JkZXIpLiAKCiMjIFBsYW4KClRoZSBmb2xsb3dpbmcgcHJvY2VkdXJlcyB3ZXJlIGNvbmR1Y3RlZDoKLSBwbGFpbiBjaGVzdCB4LXJheSAocHJvY2VkdXJlKQpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIGFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0ClRoZSBwYXRpZW50IHdhcyBwbGFjZWQgb24gYSBjYXJlcGxhbjoKLSByZXNwaXJhdG9yeSB0aGVyYXB5Cg=="}}],"context":{"encounter":[{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"}],"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"}}} +{"resourceType":"DocumentReference","id":"91115f6d-efb5-4fdc-9ef9-63fe03e21c2a","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2015-04-23T14:15:05.225-07:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTUtMDQtMjMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUsIGhlcCBiLCBhZHVsdC4gCg=="}}],"context":{"encounter":[{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"}],"period":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"}}} +{"resourceType":"DocumentReference","id":"d9054dcc-9a9e-480e-b5dd-3d6a54429c22","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2016-02-13T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTYtMDItMTMKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDIzIHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIG5hdGF6aWEgMjggZGF5IHBhY2sK"}}],"context":{"encounter":[{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"}],"period":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"f77b1642-1b8d-40b1-a1f0-2ff8053d2966","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2016-10-30T14:15:05.225-07:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTYtMTAtMzAKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI0IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBzaW5nbGUuIFBhdGllbnQgaXMgYW4gYWN0aXZlIHNtb2tlciBhbmQgaXMgYW4gYWxjb2hvbGljLiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4gUGF0aWVudCBoYXMgYSBoaWdoIHNjaG9vbCBlZHVjYXRpb24uIFBhdGllbnQgY3VycmVudGx5IGhhcyBNZWRpY2FpZC4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKYWNldGFtaW5vcGhlbiAzMjUgbWcgb3JhbCB0YWJsZXQ7IG5hdGF6aWEgMjggZGF5IHBhY2s7IDE2OCBociBldGhpbnlsIGVzdHJhZGlvbCAwLjAwMTQ2IG1nL2hyIC8gbm9yZWxnZXN0cm9taW4gMC4wMDYyNSBtZy9ociB0cmFuc2Rlcm1hbCBzeXN0ZW07IHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0OyB0cmluZXNzYSAyOCBkYXkgcGFjazsgeWF6IDI4IGRheSBwYWNrCgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggZnJhY3R1cmUgb2YgY2xhdmljbGUuIAoKIyMgUGxhbgoKVGhlIGZvbGxvd2luZyBwcm9jZWR1cmVzIHdlcmUgY29uZHVjdGVkOgotIGNsYXZpY2xlIHgtcmF5Ci0gYWRtaXNzaW9uIHRvIG9ydGhvcGVkaWMgZGVwYXJ0bWVudApUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIG1lcGVyaWRpbmUgaHlkcm9jaGxvcmlkZSA1MCBtZyBvcmFsIHRhYmxldAotIGlidXByb2ZlbiAyMDAgbWcgb3JhbCB0YWJsZXQKVGhlIHBhdGllbnQgd2FzIHBsYWNlZCBvbiBhIGNhcmVwbGFuOgotIGZyYWN0dXJlIGNhcmUK"}}],"context":{"encounter":[{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"}],"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"}}} +{"resourceType":"DocumentReference","id":"f30142f1-c8b2-4642-b353-7c1ac79eddd7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2016-12-29T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTYtMTItMjkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI0IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgo="}}],"context":{"encounter":[{"reference":"Encounter/a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"}],"period":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"84fc7c21-e344-431b-922d-ff5ed0beaf48","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2017-06-15T14:15:05.225-07:00","author":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTctMDYtMTUKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuClBhdGllbnQgd2FzIGdpdmVuIHRoZSBmb2xsb3dpbmcgaW1tdW5pemF0aW9uczogaW5mbHVlbnphLCBzZWFzb25hbCwgaW5qZWN0YWJsZSwgcHJlc2VydmF0aXZlIGZyZWUuIApUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gbWVkaWNhdGlvbiByZWNvbmNpbGlhdGlvbiAocHJvY2VkdXJlKQo="}}],"context":{"encounter":[{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"}],"period":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"}}} +{"resourceType":"DocumentReference","id":"b6a6bef9-9602-40d9-bca6-b58315624cbe","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2018-02-02T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTgtMDItMDIKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgo="}}],"context":{"encounter":[{"reference":"Encounter/eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"}],"period":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"}}} +{"resourceType":"DocumentReference","id":"c95bce8c-78b5-4569-807a-0093eec4d721","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2018-02-09T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTgtMDItMDkKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI1IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gYmlsYXRlcmFsIHR1YmFsIGxpZ2F0aW9uClRoZSBwYXRpZW50IHdhcyBwbGFjZWQgb24gYSBjYXJlcGxhbjoKLSBtaW5vciBzdXJnZXJ5IGNhcmUgbWFuYWdlbWVudCAocHJvY2VkdXJlKQo="}}],"context":{"encounter":[{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"}],"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"}}} +{"resourceType":"DocumentReference","id":"2cf76961-ab88-4c68-9935-ce10c18faab6","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"status":"current","type":{"coding":[{"system":"http://loinc.org","code":"34117-2","display":"History and physical note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation+Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"date":"2018-12-11T13:15:05.225-08:00","author":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}],"content":[{"attachment":{"contentType":"text/plain","data":"CjIwMTgtMTItMTEKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkxvcml0YTIxNyBpcyBhIDI2IHllYXItb2xkIG5vbi1oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBzdHJlcHRvY29jY2FsIHNvcmUgdGhyb2F0IChkaXNvcmRlciksIGFjdXRlIGJyb25jaGl0aXMgKGRpc29yZGVyKSwgZnJhY3R1cmUgb2YgY2xhdmljbGUuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgc2luZ2xlLiBQYXRpZW50IGlzIGFuIGFjdGl2ZSBzbW9rZXIgYW5kIGlzIGFuIGFsY29ob2xpYy4gUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuIFBhdGllbnQgaGFzIGEgaGlnaCBzY2hvb2wgZWR1Y2F0aW9uLiBQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmFjZXRhbWlub3BoZW4gMzI1IG1nIG9yYWwgdGFibGV0OyBuYXRhemlhIDI4IGRheSBwYWNrOyAxNjggaHIgZXRoaW55bCBlc3RyYWRpb2wgMC4wMDE0NiBtZy9ociAvIG5vcmVsZ2VzdHJvbWluIDAuMDA2MjUgbWcvaHIgdHJhbnNkZXJtYWwgc3lzdGVtOyBwZW5pY2lsbGluIHYgcG90YXNzaXVtIDUwMCBtZyBvcmFsIHRhYmxldDsgaWJ1cHJvZmVuIDIwMCBtZyBvcmFsIHRhYmxldDsgdHJpbmVzc2EgMjggZGF5IHBhY2s7IHlheiAyOCBkYXkgcGFjazsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggc3RyZXB0b2NvY2NhbCBzb3JlIHRocm9hdCAoZGlzb3JkZXIpLiAKCiMjIFBsYW4KClRoZSBmb2xsb3dpbmcgcHJvY2VkdXJlcyB3ZXJlIGNvbmR1Y3RlZDoKLSB0aHJvYXQgY3VsdHVyZSAocHJvY2VkdXJlKQpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIHBlbmljaWxsaW4gdiBwb3Rhc3NpdW0gNTAwIG1nIG9yYWwgdGFibGV0Cg=="}}],"context":{"encounter":[{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"}],"period":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"}}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Encounter.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Encounter.ndjson new file mode 100644 index 000000000000..d973abf02d66 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Encounter.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"Encounter","id":"84fa1324-9220-4cf6-bf35-fce580dbb7a8","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"84fa1324-9220-4cf6-bf35-fce580dbb7a8"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"7c8b9bd7-0172-47af-bd21-6af5b0eacf08","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"7c8b9bd7-0172-47af-bd21-6af5b0eacf08"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}}],"period":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"location":[{"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}}],"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}} +{"resourceType":"Encounter","id":"2dd9110c-d725-45cf-827f-bafec9b3543f","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"2dd9110c-d725-45cf-827f-bafec9b3543f"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"f60a41fa-6fbf-4fc4-b565-488d6197e2f4","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"f60a41fa-6fbf-4fc4-b565-488d6197e2f4"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}}],"period":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"location":[{"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}}],"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}} +{"resourceType":"Encounter","id":"9ff3cb23-2995-4a1a-92ac-ff0a36dbec59","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"8627419c-d90e-43ce-9035-74578faa9e15","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"8627419c-d90e-43ce-9035-74578faa9e15"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}}],"period":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"location":[{"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}}],"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}} +{"resourceType":"Encounter","id":"ea1c03fb-e18d-4b93-8fea-79b98c095cbc","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"ea1c03fb-e18d-4b93-8fea-79b98c095cbc"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"12b98110-3df6-40cb-bb70-87ea01d4aa31","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"12b98110-3df6-40cb-bb70-87ea01d4aa31"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}]}],"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"47e0d3ec-0263-4fba-916e-43bf4abe166e","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"47e0d3ec-0263-4fba-916e-43bf4abe166e"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}],"text":"Encounter for check up (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"1d49703f-4c67-4a71-bb97-849b8915b718","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"1d49703f-4c67-4a71-bb97-849b8915b718"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"e32f7ba3-c22a-4222-8b53-58a3118450f7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"e32f7ba3-c22a-4222-8b53-58a3118450f7"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"EMER"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}],"text":"Encounter for 'check-up'"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}]}],"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"14349254-a133-4fac-8055-2571bc83f059","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"14349254-a133-4fac-8055-2571bc83f059"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"}}],"period":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"location":[{"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}}],"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}} +{"resourceType":"Encounter","id":"eac7e5d9-b34a-4353-81b2-5a1a40df2d7d","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"498dc8b8-db3d-4064-87eb-4011786324d5","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"498dc8b8-db3d-4064-87eb-4011786324d5"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"IMP"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}],"text":"Admission to surgical department"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Encounter","id":"dd86ac8c-7740-41f3-8dfc-7ac26e90a448","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]},"identifier":[{"use":"official","system":"https://github.com/synthetichealth/synthea","value":"dd86ac8c-7740-41f3-8dfc-7ac26e90a448"}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c","display":"Ms. Lorita217 Nolan344"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}}],"period":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"reasonCode":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"location":[{"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ExplanationOfBenefit.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ExplanationOfBenefit.ndjson new file mode 100644 index 000000000000..6a785b678a13 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ExplanationOfBenefit.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"ExplanationOfBenefit","id":"648291cb-e6ec-478a-894e-b653e7ae929b","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"a7b49375-f326-462c-91a6-fb68c3d7c359"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-05-19T14:45:05-07:00","end":"2011-05-19T14:45:05-07:00"},"created":"2010-05-19T14:45:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/a7b49375-f326-462c-91a6-fb68c3d7c359"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7"},"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-diagnosistype","code":"principal"}]}]}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"servicedPeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"}]},{"sequence":2,"diagnosisSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"servicedPeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]}}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"2400fc5c-fb9e-4703-a8b6-42dd2c50f5e5","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"d4f07a36-012e-4170-8757-0c9bfbdfa575"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-06-03T14:45:05-07:00","end":"2011-06-03T14:45:05-07:00"},"created":"2010-06-03T14:45:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"claim":{"reference":"Claim/d4f07a36-012e-4170-8757-0c9bfbdfa575"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"servicedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"encounter":[{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"}]},{"sequence":2,"informationSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"servicedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"servicedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":504.83,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":100.96600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":403.86400000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":504.83,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":504.83,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":516.2800000000001,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"96d4c306-c1c8-4619-a79c-45c9d495beec","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"6556aaa0-bf9f-40e7-bc54-1c3ceec565c7"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-03-11T13:30:05-08:00","end":"2012-03-11T13:30:05-07:00"},"created":"2011-03-11T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/6556aaa0-bf9f-40e7-bc54-1c3ceec565c7"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"servicedPeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"b8290c3e-815f-4e76-89a0-d245fc1f34f5","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"d8f548d1-bf09-4fb3-a0d0-5454feee2396"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-06-09T14:30:05-07:00","end":"2012-06-09T14:30:05-07:00"},"created":"2011-06-09T14:30:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"claim":{"reference":"Claim/d8f548d1-bf09-4fb3-a0d0-5454feee2396"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"servicedPeriod":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"encounter":[{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"}]},{"sequence":2,"informationSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"servicedPeriod":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":112.41600000000001,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"d5608f86-b442-4f45-b320-c509103e46f1","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"171190a5-7d0d-45a4-99ac-0c519b6dd270"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2013-02-28T13:30:05-08:00","end":"2014-02-28T13:30:05-08:00"},"created":"2013-02-28T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/171190a5-7d0d-45a4-99ac-0c519b6dd270"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"servicedPeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"1d89ba47-1aa8-4ac4-9605-fa831c02d973","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"50387d60-9d96-475d-9719-7b9cb30071ca"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2014-06-12T14:30:05-07:00","end":"2015-06-12T14:30:05-07:00"},"created":"2014-06-12T14:30:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"claim":{"reference":"Claim/50387d60-9d96-475d-9719-7b9cb30071ca"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"servicedPeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"encounter":[{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"}]},{"sequence":2,"informationSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"servicedPeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":3,"informationSequence":[2],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"}],"text":"Td (adult) preservative free"},"servicedPeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":4,"informationSequence":[3],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"servicedPeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":5,"informationSequence":[4],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"}],"text":"meningococcal MCV4P"},"servicedPeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":449.66400000000004,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"38b3e5e7-6c5d-42b2-8e32-13481e4ca696","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"43604c79-af2f-4ea3-a871-a6efeb148686"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-02-18T13:30:05-08:00","end":"2016-02-18T13:30:05-08:00"},"created":"2015-02-18T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/43604c79-af2f-4ea3-a871-a6efeb148686"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"servicedPeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"b09fc58f-f447-4193-9557-fd08c4ff0ff0","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"5c4943b7-52fe-43cf-b800-200585136db8"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-12T14:41:05-07:00","end":"2016-04-12T14:41:05-07:00"},"created":"2015-04-12T14:41:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/5c4943b7-52fe-43cf-b800-200585136db8"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9"},"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-diagnosistype","code":"principal"}]}]}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"servicedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"}]},{"sequence":2,"diagnosisSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"},"servicedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]}},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"}],"text":"Plain chest X-ray (procedure)"},"servicedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"net":{"value":6535.58,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":1307.116,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":5228.464,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":6535.58,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":6535.58,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":5228.464,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"f534820d-1611-43ea-9fd0-5850a2f34633","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"99fc6fc4-98f4-415a-a094-f22df454163f"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-23T14:30:05-07:00","end":"2016-04-23T14:30:05-07:00"},"created":"2015-04-23T14:30:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/99fc6fc4-98f4-415a-a094-f22df454163f"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}],"text":"Encounter for check up (procedure)"},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"}]},{"sequence":2,"informationSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":3,"informationSequence":[2],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":224.83200000000002,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"ebcc7838-65a0-46cd-bb54-a00d8d224280","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"4a41154a-06aa-4bfc-94ce-b96f28f11b80"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-02-13T13:30:05-08:00","end":"2017-02-13T13:30:05-08:00"},"created":"2016-02-13T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/4a41154a-06aa-4bfc-94ce-b96f28f11b80"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"servicedPeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"1fb7d685-cf96-4660-8c12-240da51c3640","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"07ae31e4-8480-4293-b506-5e9b32e7491b"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T16:45:05-07:00","end":"2017-10-30T16:45:05-07:00"},"created":"2016-10-30T16:45:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/07ae31e4-8480-4293-b506-5e9b32e7491b"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/115098be-c26e-47d0-9cad-be9df75d7042"},"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-diagnosistype","code":"principal"}]}]}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"20","display":"Urgent Care Facility"}]},"encounter":[{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"}]},{"sequence":2,"diagnosisSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"20","display":"Urgent Care Facility"}]}},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"}],"text":"Clavicle X-ray"},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"20","display":"Urgent Care Facility"}]},"net":{"value":516.65,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":103.33,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":413.32,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":516.65,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":516.65,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":4,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"}],"text":"Admission to orthopedic department"},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"20","display":"Urgent Care Facility"}]},"net":{"value":516.65,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":103.33,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":413.32,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":516.65,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":516.65,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":826.64,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"cf6fa314-eae2-4989-b585-a41249329ee8","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"b60311e1-94d9-473c-bf5f-ea06db1dad15"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-12-29T13:30:05-08:00","end":"2017-12-29T13:30:05-08:00"},"created":"2016-12-29T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/b60311e1-94d9-473c-bf5f-ea06db1dad15"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}],"text":"Encounter for 'check-up'"},"servicedPeriod":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"315a9c6b-8ea5-4b7b-8943-5e0de6175218","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"performer":[{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"b52c9dfa-56bb-4523-9ace-38ad614ee107"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2017-06-15T14:45:05-07:00","end":"2018-06-15T14:45:05-07:00"},"created":"2017-06-15T14:45:05-07:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"},"claim":{"reference":"Claim/b52c9dfa-56bb-4523-9ace-38ad614ee107"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"},"servicedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"encounter":[{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"}]},{"sequence":2,"informationSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"servicedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":140.52,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"servicedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"19","display":"Off Campus-Outpatient Hospital"}]},"net":{"value":482.02,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":96.404,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":385.616,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":482.02,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":482.02,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":498.032,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"08f6e8ec-9017-4919-aeb3-7b659e24e5ba","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"084d3b6a-e1d4-4e25-b90b-c970fbc20eb7"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-02-02T13:30:05-08:00","end":"2019-02-02T13:30:05-08:00"},"created":"2018-02-02T13:30:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/084d3b6a-e1d4-4e25-b90b-c970fbc20eb7"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"},"servicedPeriod":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":0.0,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"dc9c5273-41ad-42ef-8dde-1c21768c0149","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"cf98de5a-9843-43c6-84c6-e6e703ffa457"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-02-10T15:15:05-08:00","end":"2019-02-10T15:15:05-08:00"},"created":"2018-02-10T15:15:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/cf98de5a-9843-43c6-84c6-e6e703ffa457"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}],"text":"Admission to surgical department"},"servicedPeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"}]},{"sequence":2,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"}],"text":"Bilateral tubal ligation"},"servicedPeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"net":{"value":9364.49,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":1872.8980000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":7491.592000000001,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":9364.49,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":9364.49,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":7491.592000000001,"currency":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"f43fb2ac-d805-4e61-97f4-415cf14814f1","contained":[{"resourceType":"ServiceRequest","id":"referral","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"performer":[{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}]},{"resourceType":"Coverage","id":"coverage","status":"active","type":{"text":"Medicaid"},"beneficiary":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"payor":[{"display":"Medicaid"}]}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"15be99a3-ce23-43fd-a32a-fbda483a1224"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claim-type","code":"institutional"}]},"use":"claim","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-12-11T13:45:05-08:00","end":"2019-12-11T13:45:05-08:00"},"created":"2018-12-11T13:45:05-08:00","insurer":{"display":"Medicaid"},"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"referral":{"reference":"#referral"},"facility":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"claim":{"reference":"Claim/15be99a3-ce23-43fd-a32a-fbda483a1224"},"outcome":"complete","careTeam":[{"sequence":1,"provider":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"role":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/9c0af3aa-42c9-43a6-8677-ae5fa3f34922"},"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-diagnosistype","code":"principal"}]}]}],"insurance":[{"focal":true,"coverage":{"reference":"#coverage","display":"Medicaid"}}],"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"},"servicedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"}]},{"sequence":2,"diagnosisSequence":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"servicedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]}},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"productOrService":{"coding":[{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"}],"text":"Throat culture (procedure)"},"servicedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/ex-serviceplace","code":"21","display":"Inpatient Hospital"}]},"net":{"value":1958.61,"currency":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":391.722,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":1566.888,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":1958.61,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":1958.61,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"currency":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"total":[{"category":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/adjudication","code":"submitted","display":"Submitted Amount"}],"text":"Submitted Amount"},"amount":{"value":129.16,"currency":"USD"}}],"payment":{"amount":{"value":1566.888,"currency":"USD"}}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ImagingStudy.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ImagingStudy.ndjson new file mode 100644 index 000000000000..ca85bc9f5ad0 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/ImagingStudy.ndjson @@ -0,0 +1 @@ +{"resourceType":"ImagingStudy","id":"91afdb24-acac-411d-995e-0d36187ea211","identifier":[{"use":"official","system":"urn:ietf:rfc:3986","value":"urn:oid:1.2.840.99999999.22502730.1589831196459"}],"status":"available","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"started":"2016-10-30T14:15:05-07:00","numberOfSeries":1,"numberOfInstances":1,"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"series":[{"uid":"1.2.840.99999999.1.65971777.1589831196459","number":1,"modality":{"system":"http://dicom.nema.org/resources/ontology/DCM","code":"DX","display":"Digital Radiography"},"numberOfInstances":1,"bodySite":{"system":"http://snomed.info/sct","code":"51299004","display":"Clavicle"},"started":"2016-10-30T14:15:05-07:00","instance":[{"uid":"1.2.840.99999999.1.1.48010377.1589831196459","sopClass":{"system":"urn:ietf:rfc:3986","code":"1.2.840.10008.5.1.4.1.1.1.1"},"number":1,"title":"Image of clavicle"}]}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Immunization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Immunization.ndjson new file mode 100644 index 000000000000..cf15293f2474 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Immunization.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"Immunization","id":"f28a9afd-a143-45db-8aeb-082b0df22426","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"occurrenceDateTime":"2010-06-03T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"e755857d-8ce3-42b6-81f0-d5b52524fae5","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"occurrenceDateTime":"2011-06-09T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"490a18a8-30e4-4a65-9a8c-0993671cf186","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"occurrenceDateTime":"2014-06-12T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"e451e36e-dc92-4fd9-a924-1a83a04da939","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"}],"text":"Td (adult) preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"occurrenceDateTime":"2014-06-12T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"673cd32a-bf63-4c7a-9e57-1b92d9fb15e9","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"occurrenceDateTime":"2014-06-12T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"41daff0e-fb2b-4216-8065-1481d7c962ed","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"}],"text":"meningococcal MCV4P"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"occurrenceDateTime":"2014-06-12T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Immunization","id":"aebc3fce-9411-4246-87ee-17a65f3f5f9a","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"occurrenceDateTime":"2015-04-23T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Immunization","id":"ab327672-36ba-4980-83a8-c4a1525de444","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"occurrenceDateTime":"2015-04-23T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Immunization","id":"bc61721c-4053-4a81-b1e5-b64194e1efa1","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization"]},"status":"completed","vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"occurrenceDateTime":"2017-06-15T14:15:05-07:00","primarySource":true,"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Location.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Location.ndjson new file mode 100644 index 000000000000..dc8773c211f9 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Location.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Location","id":"e39647c2-31b8-4e0c-b383-0994cd478ca0","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-location"]},"status":"active","name":"METROWEST MEDICAL CENTER","telecom":[{"system":"phone","value":"5083831000"}],"address":{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"},"position":{"longitude":-71.436196,"latitude":42.307905},"managingOrganization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Location","id":"7a41edc1-c724-4bf9-b850-c78f9b1d387d","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-location"]},"status":"active","name":"PCP68975","telecom":[{"system":"phone","value":"508-881-4368"}],"address":{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"},"position":{"longitude":-71.473526,"latitude":42.257754999999996},"managingOrganization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/MedicationRequest.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/MedicationRequest.ndjson new file mode 100644 index 000000000000..2e45acdbad98 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/MedicationRequest.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"MedicationRequest","id":"8a21a17d-b0b3-48b2-8427-5eb9d1138628","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"},"authoredOn":"2010-05-19T14:15:05-07:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"reasonReference":[{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7"}]} +{"resourceType":"MedicationRequest","id":"c9954767-c8b7-45e0-8b07-099351dcd9a0","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"751905","display":"Trinessa 28 Day Pack"}],"text":"Trinessa 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"},"authoredOn":"2011-03-11T13:15:05-08:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}} +{"resourceType":"MedicationRequest","id":"d10f9ae0-ffef-4a9b-8021-74a6cd6b1ad5","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"748856","display":"Yaz 28 Day Pack"}],"text":"Yaz 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"},"authoredOn":"2013-02-28T13:15:05-08:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}} +{"resourceType":"MedicationRequest","id":"c316061f-f9ad-4a30-80cf-c488b64b1112","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"1534809","display":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"}],"text":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"},"authoredOn":"2015-02-18T13:15:05-08:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}} +{"resourceType":"MedicationRequest","id":"4b2bf1a7-6801-4ed0-85d2-336106458fdc","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"313782","display":"Acetaminophen 325 MG Oral Tablet"}],"text":"Acetaminophen 325 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"authoredOn":"2015-04-12T14:15:05-07:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"reasonReference":[{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9"}]} +{"resourceType":"MedicationRequest","id":"1df5e2a6-88ef-423c-ae94-5972e6ed7d36","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"978950","display":"Natazia 28 Day Pack"}],"text":"Natazia 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"},"authoredOn":"2016-02-13T13:15:05-08:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"}} +{"resourceType":"MedicationRequest","id":"7202acd4-409c-4078-87bb-125f9ffe4bad","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"861467","display":"Meperidine Hydrochloride 50 MG Oral Tablet"}],"text":"Meperidine Hydrochloride 50 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"authoredOn":"2016-10-30T14:15:05-07:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"dosageInstruction":[{"sequence":1,"timing":{"repeat":{"frequency":1,"period":4.0,"periodUnit":"h"}},"asNeededBoolean":false,"doseAndRate":[{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/dose-rate-type","code":"ordered","display":"Ordered"}]},"doseQuantity":{"value":1.0}}]}]} +{"resourceType":"MedicationRequest","id":"edfc50e7-5843-4cbe-b4d2-b8a0c5b6bcfe","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"310965","display":"Ibuprofen 200 MG Oral Tablet"}],"text":"Ibuprofen 200 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"authoredOn":"2016-10-30T14:15:05-07:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"dosageInstruction":[{"sequence":1,"asNeededBoolean":true}]} +{"resourceType":"MedicationRequest","id":"86aaf462-e2c8-4cdd-8db1-5edb78f15ea1","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-medicationrequest"]},"status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},"authoredOn":"2018-12-11T13:15:05-08:00","requester":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"reasonReference":[{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7"},{"reference":"Condition/9c0af3aa-42c9-43a6-8677-ae5fa3f34922"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Observation.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Observation.ndjson new file mode 100644 index 000000000000..e81bd862bb17 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Observation.ndjson @@ -0,0 +1,54 @@ +{"resourceType":"Observation","id":"df86f58b-b6bb-43cf-9a62-623c1f5bc51c","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyheight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":153.6,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"06a121ab-df4b-4336-833b-e36bd44f5387","status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":1,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"379d9202-c0c8-46ee-8a27-edac176ff215","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyweight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":55.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"9c8009e2-8704-4150-b33b-5b43eaea568c","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bmi","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":23.41,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"63584063-b54c-4363-a1b2-21ab1ec06d60","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/pediatric-bmi-for-age"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":71.788,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"dbe27317-6771-4cfd-a250-d6260ad4da83","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":72,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":1.2E+2,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"0df769ad-ca90-4020-b214-e30665499e31","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/heartrate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"0f7a3b4c-9919-4642-85f1-37538e1a6018","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/resprate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"470a3225-f3e1-47f6-bd23-ac09b6f3b0c2","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"ff702f77-3aeb-44a9-91bb-e663930ba12d","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyheight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":153.7,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"66c84e05-25e6-426c-9d5e-7bef76a79a7e","status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"5ebce2f1-3136-4318-9ffb-acfe4ecf0a92","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyweight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":62.1,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"bcce2383-307d-420c-a14a-f4e2d0969093","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bmi","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":26.3,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"5f3c6fbb-024c-4119-94fb-f6c603bd2834","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/pediatric-bmi-for-age"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":85.491,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"a962c9cc-5c29-4e3f-91df-369aa539a2fe","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":88,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":121,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"c8306162-fcde-4a60-aa71-7875f95c8c9a","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/heartrate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":93,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"81faaa81-ea0c-4c7c-aaec-cb05a354d6a9","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/resprate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"759df675-5dfc-4e52-83f6-55bbc7f099c4","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"20c3ca50-9aaa-4b11-b202-66976092df3f","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyheight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"b1d50c74-8e57-4086-88fa-74e62885be4e","status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"992a5862-97d3-4b37-8100-c6a906b0c819","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyweight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":69.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"ea97bfc2-c5b8-4b59-b1e4-4b72a94269f9","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bmi","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":29.25,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"7a97a0ca-2d18-427c-afda-68c6140b7359","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":87,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":122,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"ae5fc42b-2535-47a8-a180-2494487a8281","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/heartrate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":9E+1,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"8e9f3656-5bea-4473-b8b3-414f6d8c0c49","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/resprate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":16,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"4b4e7514-1472-4455-b30d-9632803a4094","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"6690-2","display":"Leukocytes [#/volume] in Blood by Automated count"}],"text":"Leukocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":7.2857,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"53a4daa2-d54e-4ae0-bb11-5a808163f42c","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"789-8","display":"Erythrocytes [#/volume] in Blood by Automated count"}],"text":"Erythrocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":4.7109,"unit":"10*6/uL","system":"http://unitsofmeasure.org","code":"10*6/uL"}} +{"resourceType":"Observation","id":"a6a99aca-ef13-449d-9354-e009d08e8c71","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"718-7","display":"Hemoglobin [Mass/volume] in Blood"}],"text":"Hemoglobin [Mass/volume] in Blood"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":15.964,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"d60aaa5a-2632-494e-ba83-794ad85127d5","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"4544-3","display":"Hematocrit [Volume Fraction] of Blood by Automated count"}],"text":"Hematocrit [Volume Fraction] of Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":45.934,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"c25c4744-6c81-4a1f-91f8-0f68ddf1f4af","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"787-2","display":"MCV [Entitic volume] by Automated count"}],"text":"MCV [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":94.793,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"ff785a7c-4966-4df7-a878-0dcb6de76abb","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"785-6","display":"MCH [Entitic mass] by Automated count"}],"text":"MCH [Entitic mass] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":28.937,"unit":"pg","system":"http://unitsofmeasure.org","code":"pg"}} +{"resourceType":"Observation","id":"c8b433eb-f658-44ae-b7d7-fa0d62cb701a","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"786-4","display":"MCHC [Mass/volume] by Automated count"}],"text":"MCHC [Mass/volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":33.479,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"69bcf0d7-7b07-4458-9ff6-788dc2443246","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"21000-5","display":"Erythrocyte distribution width [Entitic volume] by Automated count"}],"text":"Erythrocyte distribution width [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":43.303,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"fc5546f2-b5a9-44db-ab27-f04194526e5b","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"777-3","display":"Platelets [#/volume] in Blood by Automated count"}],"text":"Platelets [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":173.89,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"90029698-691a-430e-bca8-6f54338193fc","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"32207-3","display":"Platelet distribution width [Entitic volume] in Blood by Automated count"}],"text":"Platelet distribution width [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":413.95,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"48c316cf-6407-4574-b884-142bb11b0b42","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"32623-1","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}],"text":"Platelet mean volume [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":10.07,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"9ea6c443-bb4a-4e8d-a810-fd57e1cb0607","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"a494d6f8-d2ef-49dd-9673-f759e885dd33","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyheight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"40b93af4-f801-4966-b980-8dece84be05c","status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"dacd8c8a-6cd8-4c95-82d5-a89688fd42e2","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyweight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":70.3,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"8b03de48-f251-4056-b351-2cef4cde7071","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bmi","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":29.73,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"5c432044-547b-4217-9d30-7f6dc62c34f5","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":79,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":129,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"40ef3a2c-1458-44dc-a197-39c8ccfd84f5","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/heartrate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":9E+1,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"10b3fc73-9295-498b-a958-9f026898f480","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/resprate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":15,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"03d9292b-015f-4b16-bc25-a3c928a9e4f7","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"5490ec27-04dd-4e77-8e95-13334038599d","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyheight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"2c78b705-6228-43d0-a13a-e44dc653327d","status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":0,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"69a5d27f-352d-45e1-af40-6883c90a63f9","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodyweight","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66.8,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"77253638-9ca7-47f6-a7a7-40fde67593a7","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bmi","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":28.22,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"beba7f58-c49e-41e4-8f6d-5bf11107ef36","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":69,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":111,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"d4e540aa-9747-41ad-b1b6-04c972af0b38","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/heartrate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"5fb102db-15c6-4b87-a97d-44264a7ed7fc","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/resprate","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":12,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"24893400-afed-44fe-8faf-5ac5cfe8dd34","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"3989f7ef-48ee-4a8a-a2a6-422f293cad4e","meta":{"profile":["http://hl7.org/fhir/StructureDefinition/bodytemp","http://hl7.org/fhir/StructureDefinition/vitalsigns"]},"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8310-5","display":"Body temperature"},{"system":"http://loinc.org","code":"8331-1","display":"Oral temperature"}],"text":"Body temperature"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},"effectiveDateTime":"2018-12-11T13:15:05-08:00","issued":"2018-12-11T13:15:05.225-08:00","valueQuantity":{"value":39.024,"unit":"Cel","system":"http://unitsofmeasure.org","code":"Cel"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Organization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Organization.ndjson new file mode 100644 index 000000000000..e1036faf02c4 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Organization.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Organization","id":"465de31f-3098-365c-af70-48a071e1f5aa","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-organization"]},"identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"465de31f-3098-365c-af70-48a071e1f5aa"}],"active":true,"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/organization-type","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"}],"name":"METROWEST MEDICAL CENTER","telecom":[{"system":"phone","value":"5083831000"}],"address":[{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}]} +{"resourceType":"Organization","id":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-organization"]},"identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}],"active":true,"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/organization-type","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"}],"name":"PCP68975","telecom":[{"system":"phone","value":"508-881-4368"}],"address":[{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Patient.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Patient.ndjson new file mode 100644 index 000000000000..e86812afb434 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Patient.ndjson @@ -0,0 +1 @@ +{"resourceType":"Patient","id":"1416dec1-f4b1-4b48-b7f4-650e8f67499c","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"]},"text":{"status":"generated","div":"
Generated by Synthea.Version identifier: v2.5.0-265-gbd5a00e8\n . Person seed: 6732543839779682504 Population seed: 1589831189867
"},"extension":[{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-race","extension":[{"url":"ombCategory","valueCoding":{"system":"urn:oid:2.16.840.1.113883.6.238","code":"2106-3","display":"White"}},{"url":"text","valueString":"White"}]},{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity","extension":[{"url":"ombCategory","valueCoding":{"system":"urn:oid:2.16.840.1.113883.6.238","code":"2186-5","display":"Not Hispanic or Latino"}},{"url":"text","valueString":"Not Hispanic or Latino"}]},{"url":"http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName","valueString":"Leanna255 Predovic534"},{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex","valueCode":"F"},{"url":"http://hl7.org/fhir/StructureDefinition/patient-birthPlace","valueAddress":{"city":"Southbridge","state":"Massachusetts","country":"US"}},{"url":"http://synthetichealth.github.io/synthea/disability-adjusted-life-years","valueDecimal":0.0},{"url":"http://synthetichealth.github.io/synthea/quality-adjusted-life-years","valueDecimal":27.0}],"identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR","display":"Medical Record Number"}],"text":"Medical Record Number"},"system":"http://hospital.smarthealthit.org","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"SS","display":"Social Security Number"}],"text":"Social Security Number"},"system":"http://hl7.org/fhir/sid/us-ssn","value":"999-54-3579"},{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"DL","display":"Driver's License"}],"text":"Driver's License"},"system":"urn:oid:2.16.840.1.113883.4.3.25","value":"S99972984"},{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"PPN","display":"Passport Number"}],"text":"Passport Number"},"system":"http://standardhealthrecord.org/fhir/StructureDefinition/passportNumber","value":"X39621585X"}],"name":[{"use":"official","family":"Nolan344","given":["Lorita217"],"prefix":["Ms."]}],"telecom":[{"system":"phone","value":"555-817-6998","use":"home"}],"gender":"female","birthDate":"1992-04-09","address":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/geolocation","extension":[{"url":"latitude","valueDecimal":42.27693107900605},{"url":"longitude","valueDecimal":-71.45741653702677}]}],"line":["330 Sawayn Parade"],"city":"Framingham","state":"MA","country":"US"}],"maritalStatus":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v3-MaritalStatus","code":"S","display":"S"}],"text":"S"},"multipleBirthBoolean":false,"communication":[{"language":{"coding":[{"system":"urn:ietf:bcp:47","code":"en-US","display":"English"}],"text":"English"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Practitioner.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Practitioner.ndjson new file mode 100644 index 000000000000..1ff54077e908 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Practitioner.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Practitioner","id":"c16820ae-2954-32d4-863c-e9ceb741154c","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner"]},"identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"530"}],"active":true,"name":[{"family":"Murphy561","given":["Mari763"],"prefix":["Dr."]}],"telecom":[{"extension":[{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-direct","valueBoolean":true}],"system":"email","value":"Mari763.Murphy561@example.com","use":"work"}],"address":[{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}],"gender":"female"} +{"resourceType":"Practitioner","id":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitioner"]},"identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"35750"}],"active":true,"name":[{"family":"Hilpert278","given":["Cathryn51"],"prefix":["Dr."]}],"telecom":[{"extension":[{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-direct","valueBoolean":true}],"system":"email","value":"Cathryn51.Hilpert278@example.com","use":"work"}],"address":[{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}],"gender":"female"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/PractitionerRole.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/PractitionerRole.ndjson new file mode 100644 index 000000000000..72bd745ca8d0 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/PractitionerRole.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"PractitionerRole","id":"47501a3e-8eaa-4432-9b47-750769e4bf56","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitionerrole"]},"practitioner":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c","display":"Dr. Mari763 Murphy561"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa","display":"METROWEST MEDICAL CENTER"},"code":[{"coding":[{"system":"http://nucc.org/provider-taxonomy","code":"208D00000X","display":"General Practice"}],"text":"General Practice"}],"specialty":[{"coding":[{"system":"http://nucc.org/provider-taxonomy","code":"208D00000X","display":"General Practice"}],"text":"General Practice"}],"location":[{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}],"telecom":[{"system":"phone","value":"5083831000"}]} +{"resourceType":"PractitionerRole","id":"af8e91ae-aedf-419e-bd06-cfcf8a1c612f","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-practitionerrole"]},"practitioner":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","display":"Dr. Cathryn51 Hilpert278"},"organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"},"code":[{"coding":[{"system":"http://nucc.org/provider-taxonomy","code":"208D00000X","display":"General Practice"}],"text":"General Practice"}],"specialty":[{"coding":[{"system":"http://nucc.org/provider-taxonomy","code":"208D00000X","display":"General Practice"}],"text":"General Practice"}],"location":[{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}],"telecom":[{"system":"phone","value":"508-881-4368"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Procedure.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Procedure.ndjson new file mode 100644 index 000000000000..6bda489a5287 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Procedure.ndjson @@ -0,0 +1,7 @@ +{"resourceType":"Procedure","id":"eba3e367-30ea-418d-ae97-e91506dea85f","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},"performedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:30:05-07:00"},"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Procedure","id":"78d69a34-6c80-4a6c-abbb-0accca0a17a5","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"}],"text":"Plain chest X-ray (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},"performedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:26:05-07:00"},"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"reasonReference":[{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9","display":"Acute bronchitis (disorder)"}]} +{"resourceType":"Procedure","id":"e614b129-913f-42ac-bcc3-266db50af307","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"}],"text":"Clavicle X-ray"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T14:45:05-07:00"},"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Procedure","id":"cf1cfcd7-ca27-4bf1-b8f0-14e31c69824e","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"}],"text":"Admission to orthopedic department"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T15:15:05-07:00"},"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"reasonReference":[{"reference":"Condition/115098be-c26e-47d0-9cad-be9df75d7042","display":"Fracture of clavicle"}]} +{"resourceType":"Procedure","id":"ea19efb7-6346-45e9-aa8b-3195cc0b8aee","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},"performedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:30:05-07:00"},"location":{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d","display":"PCP68975"}} +{"resourceType":"Procedure","id":"91112614-fd2f-46cb-ad49-96111084ef1c","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"}],"text":"Bilateral tubal ligation"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"},"performedPeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-09T15:15:05-08:00"},"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"}} +{"resourceType":"Procedure","id":"1b7f052f-0bb7-422e-86c9-0c8f6af49fe2","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure"]},"status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"}],"text":"Throat culture (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},"performedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:30:05-08:00"},"location":{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0","display":"METROWEST MEDICAL CENTER"},"reasonReference":[{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7","display":"Streptococcal sore throat (disorder)"},{"reference":"Condition/9c0af3aa-42c9-43a6-8677-ae5fa3f34922","display":"Streptococcal sore throat (disorder)"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Provenance.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Provenance.ndjson new file mode 100644 index 000000000000..736f152ede70 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/R4/Provenance.ndjson @@ -0,0 +1 @@ +{"resourceType":"Provenance","id":"71dac48d-2e1b-4fce-8615-0d1c27a0be2d","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-provenance"]},"target":[{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"reference":"Location/e39647c2-31b8-4e0c-b383-0994cd478ca0"},{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},{"reference":"PractitionerRole/47501a3e-8eaa-4432-9b47-750769e4bf56"},{"reference":"Encounter/84fa1324-9220-4cf6-bf35-fce580dbb7a8"},{"reference":"Condition/4336ae40-60ee-4148-9271-a372cb9600a7"},{"reference":"MedicationRequest/8a21a17d-b0b3-48b2-8427-5eb9d1138628"},{"reference":"Claim/e24412c9-d798-4a45-8d29-98027d1eaa5d"},{"reference":"DiagnosticReport/c31327f8-f2ec-4b26-a8bd-40aff65cc085"},{"reference":"DocumentReference/a6e558a3-7907-4185-9537-5b3291e05639"},{"reference":"Claim/a7b49375-f326-462c-91a6-fb68c3d7c359"},{"reference":"ExplanationOfBenefit/648291cb-e6ec-478a-894e-b653e7ae929b"},{"reference":"Location/7a41edc1-c724-4bf9-b850-c78f9b1d387d"},{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"},{"reference":"PractitionerRole/af8e91ae-aedf-419e-bd06-cfcf8a1c612f"},{"reference":"Encounter/7c8b9bd7-0172-47af-bd21-6af5b0eacf08"},{"reference":"Observation/df86f58b-b6bb-43cf-9a62-623c1f5bc51c"},{"reference":"Observation/06a121ab-df4b-4336-833b-e36bd44f5387"},{"reference":"Observation/379d9202-c0c8-46ee-8a27-edac176ff215"},{"reference":"Observation/9c8009e2-8704-4150-b33b-5b43eaea568c"},{"reference":"Observation/63584063-b54c-4363-a1b2-21ab1ec06d60"},{"reference":"Observation/dbe27317-6771-4cfd-a250-d6260ad4da83"},{"reference":"Observation/0df769ad-ca90-4020-b214-e30665499e31"},{"reference":"Observation/0f7a3b4c-9919-4642-85f1-37538e1a6018"},{"reference":"Observation/470a3225-f3e1-47f6-bd23-ac09b6f3b0c2"},{"reference":"Procedure/eba3e367-30ea-418d-ae97-e91506dea85f"},{"reference":"Immunization/f28a9afd-a143-45db-8aeb-082b0df22426"},{"reference":"DiagnosticReport/08e52e51-ee85-4e52-a192-27d6d817334b"},{"reference":"DocumentReference/9e6df167-efc0-4b0a-922a-11afba4fe703"},{"reference":"Claim/d4f07a36-012e-4170-8757-0c9bfbdfa575"},{"reference":"ExplanationOfBenefit/2400fc5c-fb9e-4703-a8b6-42dd2c50f5e5"},{"reference":"Encounter/2dd9110c-d725-45cf-827f-bafec9b3543f"},{"reference":"MedicationRequest/c9954767-c8b7-45e0-8b07-099351dcd9a0"},{"reference":"Claim/e4ae0e04-c0b5-47c0-9f1c-1a4cefe48a97"},{"reference":"DiagnosticReport/e00fc1e7-2625-4515-8cdf-9b22d41ae0e6"},{"reference":"DocumentReference/a2a936d2-11dc-46b2-81d2-4f85ba5784ed"},{"reference":"Claim/6556aaa0-bf9f-40e7-bc54-1c3ceec565c7"},{"reference":"ExplanationOfBenefit/96d4c306-c1c8-4619-a79c-45c9d495beec"},{"reference":"Encounter/f60a41fa-6fbf-4fc4-b565-488d6197e2f4"},{"reference":"Observation/ff702f77-3aeb-44a9-91bb-e663930ba12d"},{"reference":"Observation/66c84e05-25e6-426c-9d5e-7bef76a79a7e"},{"reference":"Observation/5ebce2f1-3136-4318-9ffb-acfe4ecf0a92"},{"reference":"Observation/bcce2383-307d-420c-a14a-f4e2d0969093"},{"reference":"Observation/5f3c6fbb-024c-4119-94fb-f6c603bd2834"},{"reference":"Observation/a962c9cc-5c29-4e3f-91df-369aa539a2fe"},{"reference":"Observation/c8306162-fcde-4a60-aa71-7875f95c8c9a"},{"reference":"Observation/81faaa81-ea0c-4c7c-aaec-cb05a354d6a9"},{"reference":"Observation/759df675-5dfc-4e52-83f6-55bbc7f099c4"},{"reference":"Immunization/e755857d-8ce3-42b6-81f0-d5b52524fae5"},{"reference":"DiagnosticReport/c76085ce-ced6-4fb2-8a88-1c62638db794"},{"reference":"DocumentReference/246b40b8-e424-4be1-b296-de4e7c10e52d"},{"reference":"Claim/d8f548d1-bf09-4fb3-a0d0-5454feee2396"},{"reference":"ExplanationOfBenefit/b8290c3e-815f-4e76-89a0-d245fc1f34f5"},{"reference":"Encounter/9ff3cb23-2995-4a1a-92ac-ff0a36dbec59"},{"reference":"MedicationRequest/d10f9ae0-ffef-4a9b-8021-74a6cd6b1ad5"},{"reference":"Claim/2d2de9d8-9f05-4979-b19f-62d0733cbe9b"},{"reference":"DiagnosticReport/124c584e-a411-4ef2-8a28-875b40bdffaf"},{"reference":"DocumentReference/99076549-16bc-4843-b6b2-1a912336a5a9"},{"reference":"Claim/171190a5-7d0d-45a4-99ac-0c519b6dd270"},{"reference":"ExplanationOfBenefit/d5608f86-b442-4f45-b320-c509103e46f1"},{"reference":"Encounter/8627419c-d90e-43ce-9035-74578faa9e15"},{"reference":"Observation/20c3ca50-9aaa-4b11-b202-66976092df3f"},{"reference":"Observation/b1d50c74-8e57-4086-88fa-74e62885be4e"},{"reference":"Observation/992a5862-97d3-4b37-8100-c6a906b0c819"},{"reference":"Observation/ea97bfc2-c5b8-4b59-b1e4-4b72a94269f9"},{"reference":"Observation/7a97a0ca-2d18-427c-afda-68c6140b7359"},{"reference":"Observation/ae5fc42b-2535-47a8-a180-2494487a8281"},{"reference":"Observation/8e9f3656-5bea-4473-b8b3-414f6d8c0c49"},{"reference":"Observation/4b4e7514-1472-4455-b30d-9632803a4094"},{"reference":"Observation/53a4daa2-d54e-4ae0-bb11-5a808163f42c"},{"reference":"Observation/a6a99aca-ef13-449d-9354-e009d08e8c71"},{"reference":"Observation/d60aaa5a-2632-494e-ba83-794ad85127d5"},{"reference":"Observation/c25c4744-6c81-4a1f-91f8-0f68ddf1f4af"},{"reference":"Observation/ff785a7c-4966-4df7-a878-0dcb6de76abb"},{"reference":"Observation/c8b433eb-f658-44ae-b7d7-fa0d62cb701a"},{"reference":"Observation/69bcf0d7-7b07-4458-9ff6-788dc2443246"},{"reference":"Observation/fc5546f2-b5a9-44db-ab27-f04194526e5b"},{"reference":"Observation/90029698-691a-430e-bca8-6f54338193fc"},{"reference":"Observation/48c316cf-6407-4574-b884-142bb11b0b42"},{"reference":"Observation/9ea6c443-bb4a-4e8d-a810-fd57e1cb0607"},{"reference":"Immunization/490a18a8-30e4-4a65-9a8c-0993671cf186"},{"reference":"Immunization/e451e36e-dc92-4fd9-a924-1a83a04da939"},{"reference":"Immunization/673cd32a-bf63-4c7a-9e57-1b92d9fb15e9"},{"reference":"Immunization/41daff0e-fb2b-4216-8065-1481d7c962ed"},{"reference":"DiagnosticReport/a5b95e67-0013-4c75-812a-4c1464642fc7"},{"reference":"DiagnosticReport/6c7aed2a-2f9f-4796-85a9-ffa36af2560b"},{"reference":"DocumentReference/f9bba222-4d5f-4b1b-b681-c4ed60c05f89"},{"reference":"Claim/50387d60-9d96-475d-9719-7b9cb30071ca"},{"reference":"ExplanationOfBenefit/1d89ba47-1aa8-4ac4-9605-fa831c02d973"},{"reference":"Encounter/ea1c03fb-e18d-4b93-8fea-79b98c095cbc"},{"reference":"MedicationRequest/c316061f-f9ad-4a30-80cf-c488b64b1112"},{"reference":"Claim/39917e96-c35b-4c72-9abf-222dacb4d19e"},{"reference":"DiagnosticReport/728336a3-9e9a-4737-8767-b8de22d14385"},{"reference":"DocumentReference/56a84077-f6ac-462b-8c81-40ec2d97b9f6"},{"reference":"Claim/43604c79-af2f-4ea3-a871-a6efeb148686"},{"reference":"ExplanationOfBenefit/38b3e5e7-6c5d-42b2-8e32-13481e4ca696"},{"reference":"Encounter/12b98110-3df6-40cb-bb70-87ea01d4aa31"},{"reference":"Condition/c8d97153-17c7-4a2b-bafd-052ff8f30eb9"},{"reference":"Procedure/78d69a34-6c80-4a6c-abbb-0accca0a17a5"},{"reference":"MedicationRequest/4b2bf1a7-6801-4ed0-85d2-336106458fdc"},{"reference":"Claim/9e1c8ca0-e622-46c2-ae92-5b32f9ebfff0"},{"reference":"CareTeam/111c8459-549e-4a2b-ab70-27f3b2408eac"},{"reference":"CarePlan/32841402-d5d8-4615-af90-473bf6f417fa"},{"reference":"DiagnosticReport/0dfbba83-7fa9-48ab-ab34-a545a161d6e8"},{"reference":"DocumentReference/f1489031-2951-45c0-8e24-975e7d3eae5d"},{"reference":"Claim/5c4943b7-52fe-43cf-b800-200585136db8"},{"reference":"ExplanationOfBenefit/b09fc58f-f447-4193-9557-fd08c4ff0ff0"},{"reference":"Encounter/47e0d3ec-0263-4fba-916e-43bf4abe166e"},{"reference":"Observation/a494d6f8-d2ef-49dd-9673-f759e885dd33"},{"reference":"Observation/40b93af4-f801-4966-b980-8dece84be05c"},{"reference":"Observation/dacd8c8a-6cd8-4c95-82d5-a89688fd42e2"},{"reference":"Observation/8b03de48-f251-4056-b351-2cef4cde7071"},{"reference":"Observation/5c432044-547b-4217-9d30-7f6dc62c34f5"},{"reference":"Observation/40ef3a2c-1458-44dc-a197-39c8ccfd84f5"},{"reference":"Observation/10b3fc73-9295-498b-a958-9f026898f480"},{"reference":"Observation/03d9292b-015f-4b16-bc25-a3c928a9e4f7"},{"reference":"Immunization/aebc3fce-9411-4246-87ee-17a65f3f5f9a"},{"reference":"Immunization/ab327672-36ba-4980-83a8-c4a1525de444"},{"reference":"DiagnosticReport/a359262a-494b-48d5-9eca-52debab2c309"},{"reference":"DocumentReference/91115f6d-efb5-4fdc-9ef9-63fe03e21c2a"},{"reference":"Claim/99fc6fc4-98f4-415a-a094-f22df454163f"},{"reference":"ExplanationOfBenefit/f534820d-1611-43ea-9fd0-5850a2f34633"},{"reference":"Encounter/1d49703f-4c67-4a71-bb97-849b8915b718"},{"reference":"MedicationRequest/1df5e2a6-88ef-423c-ae94-5972e6ed7d36"},{"reference":"Claim/8c0e3992-24e7-4c44-9c50-3f0ea1431726"},{"reference":"DiagnosticReport/ef3508ff-eb22-4144-a6b5-dd745129c055"},{"reference":"DocumentReference/d9054dcc-9a9e-480e-b5dd-3d6a54429c22"},{"reference":"Claim/4a41154a-06aa-4bfc-94ce-b96f28f11b80"},{"reference":"ExplanationOfBenefit/ebcc7838-65a0-46cd-bb54-a00d8d224280"},{"reference":"Encounter/e32f7ba3-c22a-4222-8b53-58a3118450f7"},{"reference":"Condition/115098be-c26e-47d0-9cad-be9df75d7042"},{"reference":"Procedure/e614b129-913f-42ac-bcc3-266db50af307"},{"reference":"Procedure/cf1cfcd7-ca27-4bf1-b8f0-14e31c69824e"},{"reference":"MedicationRequest/7202acd4-409c-4078-87bb-125f9ffe4bad"},{"reference":"Claim/80484d29-eded-431d-bf9b-8a917089a77e"},{"reference":"MedicationRequest/edfc50e7-5843-4cbe-b4d2-b8a0c5b6bcfe"},{"reference":"Claim/41a00b32-22ff-4fd3-a388-296e4cc77dfe"},{"reference":"CareTeam/31d5cf9a-c745-4087-8042-6a9b9babaef9"},{"reference":"CarePlan/ac2893f1-9c90-437d-852f-4cc1a0155d39"},{"reference":"ImagingStudy/91afdb24-acac-411d-995e-0d36187ea211"},{"reference":"DiagnosticReport/1a30352d-3766-47dc-b9e6-9dd65f702ab1"},{"reference":"DocumentReference/f77b1642-1b8d-40b1-a1f0-2ff8053d2966"},{"reference":"Claim/07ae31e4-8480-4293-b506-5e9b32e7491b"},{"reference":"ExplanationOfBenefit/1fb7d685-cf96-4660-8c12-240da51c3640"},{"reference":"Encounter/a6db95e4-0a32-4a9b-a1b6-9f43b6e5e9f7"},{"reference":"DiagnosticReport/0996a8ad-3dae-4574-97b8-e63bfbed0436"},{"reference":"DocumentReference/f30142f1-c8b2-4642-b353-7c1ac79eddd7"},{"reference":"Claim/b60311e1-94d9-473c-bf5f-ea06db1dad15"},{"reference":"ExplanationOfBenefit/cf6fa314-eae2-4989-b585-a41249329ee8"},{"reference":"Encounter/14349254-a133-4fac-8055-2571bc83f059"},{"reference":"Observation/5490ec27-04dd-4e77-8e95-13334038599d"},{"reference":"Observation/2c78b705-6228-43d0-a13a-e44dc653327d"},{"reference":"Observation/69a5d27f-352d-45e1-af40-6883c90a63f9"},{"reference":"Observation/77253638-9ca7-47f6-a7a7-40fde67593a7"},{"reference":"Observation/beba7f58-c49e-41e4-8f6d-5bf11107ef36"},{"reference":"Observation/d4e540aa-9747-41ad-b1b6-04c972af0b38"},{"reference":"Observation/5fb102db-15c6-4b87-a97d-44264a7ed7fc"},{"reference":"Observation/24893400-afed-44fe-8faf-5ac5cfe8dd34"},{"reference":"Procedure/ea19efb7-6346-45e9-aa8b-3195cc0b8aee"},{"reference":"Immunization/bc61721c-4053-4a81-b1e5-b64194e1efa1"},{"reference":"DiagnosticReport/dada411f-d95a-42b6-bea5-987b2da3f0a1"},{"reference":"DocumentReference/84fc7c21-e344-431b-922d-ff5ed0beaf48"},{"reference":"Claim/b52c9dfa-56bb-4523-9ace-38ad614ee107"},{"reference":"ExplanationOfBenefit/315a9c6b-8ea5-4b7b-8943-5e0de6175218"},{"reference":"Encounter/eac7e5d9-b34a-4353-81b2-5a1a40df2d7d"},{"reference":"DiagnosticReport/a6d08a29-a694-482d-8df8-36b63c8e8296"},{"reference":"DocumentReference/b6a6bef9-9602-40d9-bca6-b58315624cbe"},{"reference":"Claim/084d3b6a-e1d4-4e25-b90b-c970fbc20eb7"},{"reference":"ExplanationOfBenefit/08f6e8ec-9017-4919-aeb3-7b659e24e5ba"},{"reference":"Encounter/498dc8b8-db3d-4064-87eb-4011786324d5"},{"reference":"Procedure/91112614-fd2f-46cb-ad49-96111084ef1c"},{"reference":"CareTeam/e1ab345a-81bd-461e-921d-9eae42c47f8a"},{"reference":"CarePlan/5a1fb9d1-310b-4fe3-b7c3-6370529400b8"},{"reference":"DiagnosticReport/e1385d55-31f2-4a8b-8356-22869d42d0bd"},{"reference":"DocumentReference/c95bce8c-78b5-4569-807a-0093eec4d721"},{"reference":"Claim/cf98de5a-9843-43c6-84c6-e6e703ffa457"},{"reference":"ExplanationOfBenefit/dc9c5273-41ad-42ef-8dde-1c21768c0149"},{"reference":"Encounter/dd86ac8c-7740-41f3-8dfc-7ac26e90a448"},{"reference":"Condition/9c0af3aa-42c9-43a6-8677-ae5fa3f34922"},{"reference":"Observation/3989f7ef-48ee-4a8a-a2a6-422f293cad4e"},{"reference":"Procedure/1b7f052f-0bb7-422e-86c9-0c8f6af49fe2"},{"reference":"MedicationRequest/86aaf462-e2c8-4cdd-8db1-5edb78f15ea1"},{"reference":"Claim/bfc5accc-20be-4556-b76a-e076fc4b2278"},{"reference":"DiagnosticReport/6d6cb707-6e0f-4fe3-a047-8b0b0da44807"},{"reference":"DocumentReference/2cf76961-ab88-4c68-9935-ce10c18faab6"},{"reference":"Claim/15be99a3-ce23-43fd-a32a-fbda483a1224"},{"reference":"ExplanationOfBenefit/f43fb2ac-d805-4e61-97f4-415cf14814f1"}],"recorded":"2020-05-21T14:15:05.225-07:00","agent":[{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/provenance-participant-type","code":"author","display":"Author"}],"text":"Author"},"who":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","display":"PCP68975"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/CarePlan.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/CarePlan.ndjson new file mode 100644 index 000000000000..99dec1a97ca0 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/CarePlan.ndjson @@ -0,0 +1,3 @@ +{"resourceType":"CarePlan","id":"e1162c3b-86cf-4097-a33d-34bb9a93cfb9","status":"completed","intent":"order","category":[{"coding":[{"system":"http://snomed.info/sct","code":"53950000","display":"Respiratory therapy"}],"text":"Respiratory therapy"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"},"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-19T14:15:05-07:00"},"addresses":[{"reference":"Condition/5953e3df-ed6c-4509-8d56-04c745817bd0"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"304510005","display":"Recommendation to avoid exercise"}],"text":"Recommendation to avoid exercise"},"status":"completed"}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"371605008","display":"Deep breathing and coughing exercises"}],"text":"Deep breathing and coughing exercises"},"status":"completed"}}]} +{"resourceType":"CarePlan","id":"04330c15-061f-494c-9d2f-4e42eca9e099","status":"completed","intent":"order","category":[{"coding":[{"system":"http://snomed.info/sct","code":"385691007","display":"Fracture care"}],"text":"Fracture care"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-12-29T13:15:05-08:00"},"addresses":[{"reference":"Condition/0aa16b8a-bce6-4362-8312-879d7ccf29f8"}],"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed"}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"408580007","display":"Physical activity target light exercise"}],"text":"Physical activity target light exercise"},"status":"completed"}}]} +{"resourceType":"CarePlan","id":"28ffafe4-9be5-4eb5-b663-c22d10c82cf0","status":"completed","intent":"order","category":[{"coding":[{"system":"http://snomed.info/sct","code":"737471002","display":"Minor surgery care management (procedure)"}],"text":"Minor surgery care management (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/89c75588-d3a0-4ff8-8202-665ff4349fa7"},"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-23T13:15:05-08:00"},"activity":[{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"183051005","display":"Recommendation to rest"}],"text":"Recommendation to rest"},"status":"completed"}},{"detail":{"code":{"coding":[{"system":"http://snomed.info/sct","code":"243077000","display":"Recommendation to limit sexual activity"}],"text":"Recommendation to limit sexual activity"},"status":"completed"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Claim.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Claim.ndjson new file mode 100644 index 000000000000..7c94153bf3d3 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Claim.ndjson @@ -0,0 +1,25 @@ +{"resourceType":"Claim","id":"a88729f5-c37e-4949-85a7-a4a69ffedecb","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/df4ce22a-1ae5-429f-8d76-5a503a3921ab"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/7b569404-f475-45d9-94d3-b4e31fece687"}]}],"total":{"value":10.45,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"462a2021-f2c0-45cc-85ec-3338664ebd9e","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/f8fe7590-249c-4b93-8bf0-275f2feb6b81"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/7b569404-f475-45d9-94d3-b4e31fece687"}]},{"sequence":2,"diagnosisLinkId":[1]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"bf23b373-969f-4bc7-89a2-b91d321f54d1","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"information":[{"sequence":1,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/293b214e-ae9b-4053-8ed1-becbe613de18"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/63702bbb-e289-4a80-9f1a-3ef5a7d15019"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"}]},{"sequence":2,"informationLinkId":[1],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"procedureLinkId":[1],"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"430193006"}]},"net":{"value":504.83,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"94f416b6-2750-401d-af06-0f3391518cf2","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/8c7b34ec-760b-412b-bfd4-ec560b2ac998"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/ebe5ce5b-57bb-4368-93d6-0b7a0da073a1"}]}],"total":{"value":31.0,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"2bf9a87f-6c7e-4f56-b63f-4a5675faefb7","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/ebe5ce5b-57bb-4368-93d6-0b7a0da073a1"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"0dd76242-6c52-45ce-9e13-84e635c25f4a","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"information":[{"sequence":1,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/c04ae3a8-1678-404b-8fb7-2d249ba82839"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"}]},{"sequence":2,"informationLinkId":[1],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"527cff9c-e3d1-462e-8fcc-1058cf6955f1","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/546161c7-df48-41ae-a25f-2dbbf39729be"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/6245224b-6d15-448a-8e56-db606acf21d8"}]}],"total":{"value":26.21,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"7f84d563-6fb1-4339-8432-766672b1737b","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/6245224b-6d15-448a-8e56-db606acf21d8"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"9e3e2f01-530d-40b1-9795-eba32a97158b","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"information":[{"sequence":1,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/60ab5005-b572-498c-9e32-a5acdc26e3f8"}},{"sequence":2,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/dc26c48b-cdd3-4384-a0f2-02d25682a02d"}},{"sequence":3,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/52ebf7c5-8af2-4f91-9e9a-8b54de12f857"}},{"sequence":4,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/ded2b0fb-56ed-4245-85c8-0b2a02ff4b53"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"}]},{"sequence":2,"informationLinkId":[1],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"informationLinkId":[2],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":4,"informationLinkId":[3],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":5,"informationLinkId":[4],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"384aade3-08c4-4723-b9f0-029501cb4453","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/c9970c0c-236e-4431-bfe9-c08cdd622411"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/7b2e1b32-79cc-412f-b8e7-97997da45089"}]}],"total":{"value":45.32,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"0623afa7-5753-450b-9124-c93df6a8a4f7","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/7b2e1b32-79cc-412f-b8e7-97997da45089"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"f1a3cb88-268b-4308-aaa1-b7cc8f429d43","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/6cb4c447-aa9d-4bfc-89f3-e424dab1fd2a"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"}]}],"total":{"value":5.32,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"abafc9ba-edc7-4eff-98fc-e41d8785c27b","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/5953e3df-ed6c-4509-8d56-04c745817bd0"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/bbedb163-5b9d-4865-bd9f-d59f7a9c21ad"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"}]},{"sequence":2,"diagnosisLinkId":[1]},{"sequence":3,"procedureLinkId":[1],"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"399208008"}]},"net":{"value":6535.58,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"9bd3bd28-4e88-4d8a-8510-ea791953455b","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"information":[{"sequence":1,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/8ea94a08-14b2-4b18-9f70-7d5f8ae729f7"}},{"sequence":2,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/8ca4d799-58a3-4930-a35c-c6f5a1555173"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"}]},{"sequence":2,"informationLinkId":[1],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"informationLinkId":[2],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"73babc33-4afd-401f-a275-3d1e98c6007e","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/e42328cf-bb27-470f-9cff-a9dcb8fd7485"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/96baca61-8639-457b-bd48-fa32511b5985"}]}],"total":{"value":31.25,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"36c0e892-2f4c-4387-8587-7fe004a6bafc","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/96baca61-8639-457b-bd48-fa32511b5985"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"aecbb5b4-1438-4bc3-a845-dd64a9868765","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/b8c23eb0-d9b6-4413-8293-62c13b47e2ab"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"}]}],"total":{"value":113.81,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"e069f335-0774-4bf9-b4e0-821ab7c5908e","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/22ba50dd-b723-4771-9247-2b25e566da68"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"}]}],"total":{"value":17.35,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"d69ba5dd-6da7-4d7b-8cce-63b1ed0c5134","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/0aa16b8a-bce6-4362-8312-879d7ccf29f8"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/565fedff-5363-4938-9b49-1a2b5b004e15"}},{"sequence":2,"procedureReference":{"reference":"Procedure/faae5e23-a03a-450a-ad12-50ca56eaa0e1"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"}]},{"sequence":2,"diagnosisLinkId":[1]},{"sequence":3,"procedureLinkId":[1],"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"168594001"}]},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":4,"procedureLinkId":[2],"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"305428000"}]},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"4cb0502f-3892-41f6-ac59-49d31dc28193","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/522f6ab4-fd6e-42d4-b7c1-c25c99e35d3a"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"f7183ce1-f163-40b3-8ed8-0bcff268937f","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"organization":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"},"information":[{"sequence":1,"category":{"coding":[{"system":"http://hl7.org/fhir/claiminformationcategory","code":"info"}]},"valueReference":{"reference":"Immunization/69152acf-13e8-4265-bd5f-7a2df80a08f2"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/2b4a074e-e62c-4208-9ae5-be371b9208d3"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"}]},{"sequence":2,"informationLinkId":[1],"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"sequence":3,"procedureLinkId":[1],"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"430193006"}]},"net":{"value":482.02,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"e72892e7-dbe8-45ce-b6bf-4d0952c618e3","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/6fb5a1fb-c93c-4648-bb60-db870634bbf2"}]}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"8d5d5189-0c2c-4bf9-930a-7bc245908702","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/28e04a1a-c56a-4aca-a144-ec986f09f1aa"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/89c75588-d3a0-4ff8-8202-665ff4349fa7"}]},{"sequence":2,"procedureLinkId":[1],"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"287664005"}]},"net":{"value":9364.49,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"6332d509-9603-41af-905d-f1961edd19d3","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"prescription":{"reference":"MedicationRequest/f313632b-f3da-4507-a3de-954c156bd799"},"item":[{"sequence":1,"encounter":[{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"}]}],"total":{"value":16.98,"system":"urn:iso:std:iso:4217","code":"USD"}} +{"resourceType":"Claim","id":"d220ef38-fff4-45e8-94e0-da6716f84ff6","status":"active","use":"complete","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"organization":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"},"diagnosis":[{"sequence":1,"diagnosisReference":{"reference":"Condition/53dac70f-c5cc-4880-a96e-fdb157db8e3b"}}],"procedure":[{"sequence":1,"procedureReference":{"reference":"Procedure/17543a9f-3947-4a3c-af28-21391ee2674a"}}],"item":[{"sequence":1,"encounter":[{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"}]},{"sequence":2,"diagnosisLinkId":[1]},{"sequence":3,"procedureLinkId":[1],"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"117015009"}]},"net":{"value":1958.61,"system":"urn:iso:std:iso:4217","code":"USD"}}],"total":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Condition.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Condition.ndjson new file mode 100644 index 000000000000..ce06594b71d7 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Condition.ndjson @@ -0,0 +1,4 @@ +{"resourceType":"Condition","id":"f8fe7590-249c-4b93-8bf0-275f2feb6b81","clinicalStatus":"resolved","verificationStatus":"confirmed","code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/7b569404-f475-45d9-94d3-b4e31fece687"},"onsetDateTime":"2010-05-19T14:15:05-07:00","abatementDateTime":"2010-05-26T14:15:05-07:00","assertedDate":"2010-05-19T14:15:05-07:00"} +{"resourceType":"Condition","id":"5953e3df-ed6c-4509-8d56-04c745817bd0","clinicalStatus":"resolved","verificationStatus":"confirmed","code":{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}],"text":"Acute bronchitis (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"},"onsetDateTime":"2015-04-12T14:15:05-07:00","abatementDateTime":"2015-04-19T14:15:05-07:00","assertedDate":"2015-04-12T14:15:05-07:00"} +{"resourceType":"Condition","id":"0aa16b8a-bce6-4362-8312-879d7ccf29f8","clinicalStatus":"resolved","verificationStatus":"confirmed","code":{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}],"text":"Fracture of clavicle"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"onsetDateTime":"2016-10-30T14:15:05-07:00","abatementDateTime":"2016-12-29T13:15:05-08:00","assertedDate":"2016-10-30T14:15:05-07:00"} +{"resourceType":"Condition","id":"53dac70f-c5cc-4880-a96e-fdb157db8e3b","clinicalStatus":"resolved","verificationStatus":"confirmed","code":{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}],"text":"Streptococcal sore throat (disorder)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"},"onsetDateTime":"2018-12-11T13:15:05-08:00","abatementDateTime":"2018-12-19T13:15:05-08:00","assertedDate":"2018-12-11T13:15:05-08:00"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/DiagnosticReport.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/DiagnosticReport.ndjson new file mode 100644 index 000000000000..826daad9fed0 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/DiagnosticReport.ndjson @@ -0,0 +1 @@ +{"resourceType":"DiagnosticReport","id":"89228305-514c-4b1a-8000-cda6db77eccc","status":"final","code":{"coding":[{"system":"http://loinc.org","code":"58410-2","display":"Complete blood count (hemogram) panel - Blood by Automated count"}],"text":"Complete blood count (hemogram) panel - Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","result":[{"reference":"Observation/a71330cb-bdf0-4383-9411-0e3372da9577","display":"Leukocytes [#/volume] in Blood by Automated count"},{"reference":"Observation/f74b75ea-1ad6-4418-ae8b-2bd62e875e21","display":"Erythrocytes [#/volume] in Blood by Automated count"},{"reference":"Observation/573fb355-b0bf-4af5-98d8-ead5389eb3cc","display":"Hemoglobin [Mass/volume] in Blood"},{"reference":"Observation/d750ec32-b61a-40da-b58d-fff11de5bf27","display":"Hematocrit [Volume Fraction] of Blood by Automated count"},{"reference":"Observation/ab86a392-7cca-4625-97d4-421aeaa14382","display":"MCV [Entitic volume] by Automated count"},{"reference":"Observation/0bb6c8de-bf30-43bd-9ae7-b56bc3c74914","display":"MCH [Entitic mass] by Automated count"},{"reference":"Observation/ff40e33c-caee-4181-b71f-cf9a12f7a24f","display":"MCHC [Mass/volume] by Automated count"},{"reference":"Observation/2e6accca-09d2-462d-b7dc-777dfa52d06b","display":"Erythrocyte distribution width [Entitic volume] by Automated count"},{"reference":"Observation/0ac11784-c312-44c4-9942-b7d2beb26056","display":"Platelets [#/volume] in Blood by Automated count"},{"reference":"Observation/d29445ad-c8cf-4ced-b59a-a4683c04ce51","display":"Platelet distribution width [Entitic volume] in Blood by Automated count"},{"reference":"Observation/c80a8ad6-4bf5-476e-b3ef-09d812e14ded","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Encounter.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Encounter.ndjson new file mode 100644 index 000000000000..b0bdae6dc4c0 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Encounter.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"Encounter","id":"7b569404-f475-45d9-94d3-b4e31fece687","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2010-05-19T14:15:05-07:00","end":"2010-05-19T14:45:05-07:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:45:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"ebe5ce5b-57bb-4368-93d6-0b7a0da073a1","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"3a99bba3-f304-4d33-826a-018499a08f5a","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2011-06-09T14:15:05-07:00","end":"2011-06-09T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"6245224b-6d15-448a-8e56-db606acf21d8","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"4ef1ef04-c4f7-4739-b722-e668ec9f6c35","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2014-06-12T14:15:05-07:00","end":"2014-06-12T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"7b2e1b32-79cc-412f-b8e7-97997da45089","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"37cce86a-e35b-4879-b8ed-4f22b10b2f7c","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:41:05-07:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"10509002","display":"Acute bronchitis (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"74003abb-cee0-4151-8469-4a815fff57aa","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for check up (procedure)"}],"text":"Encounter for check up (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"96baca61-8639-457b-bd48-fa32511b5985","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"187a1ecd-b5f5-4566-b39d-ae646062cfbc","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"EMER"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"50849002","display":"Emergency room admission (procedure)"}],"text":"Emergency room admission (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"522f6ab4-fd6e-42d4-b7c1-c25c99e35d3a","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185349003","display":"Encounter for 'check-up'"}],"text":"Encounter for 'check-up'"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2016-12-29T13:15:05-08:00","end":"2016-12-29T13:30:05-08:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"58150001","display":"Fracture of clavicle"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"fc81047d-ee3a-4857-b96c-0a091e957d7a","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"162673000","display":"General examination of patient (procedure)"}],"text":"General examination of patient (procedure)"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}}],"period":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:45:05-07:00"},"serviceProvider":{"reference":"Organization/58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}} +{"resourceType":"Encounter","id":"6fb5a1fb-c93c-4648-bb60-db870634bbf2","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"698314001","display":"Consultation for treatment"}],"text":"Consultation for treatment"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"89c75588-d3a0-4ff8-8202-665ff4349fa7","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"IMP"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"305408004","display":"Admission to surgical department"}],"text":"Admission to surgical department"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-10T15:15:05-08:00"},"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} +{"resourceType":"Encounter","id":"99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9","status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"AMB"},"type":[{"coding":[{"system":"http://snomed.info/sct","code":"185345009","display":"Encounter for symptom"}],"text":"Encounter for symptom"}],"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"participant":[{"individual":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"}}],"period":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:45:05-08:00"},"reason":[{"coding":[{"system":"http://snomed.info/sct","code":"43878008","display":"Streptococcal sore throat (disorder)"}]}],"serviceProvider":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ExplanationOfBenefit.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ExplanationOfBenefit.ndjson new file mode 100644 index 000000000000..7a6f130333c9 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ExplanationOfBenefit.ndjson @@ -0,0 +1,16 @@ +{"resourceType":"ExplanationOfBenefit","id":"26ef8891-a160-4c1e-8072-f974638866f6","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"462a2021-f2c0-45cc-85ec-3338664ebd9e"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2010-05-19T14:45:05-07:00","end":"2011-05-19T14:45:05-07:00"},"provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-poa-ind-sw1-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-poa-ind-sw1","code":"Y","display":"Diagnosis present at time of admission"}}],"sequence":1,"diagnosisReference":{"reference":"Condition/f8fe7590-249c-4b93-8bf0-275f2feb6b81"},"type":[{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/diagnosis-type","code":"principal"}]}]}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"a0fba859-3a95-42b3-9a74-3c46be7cfcb7","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"bf23b373-969f-4bc7-89a2-b91d321f54d1"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2010-06-03T14:45:05-07:00","end":"2011-06-03T14:45:05-07:00"},"provider":{"identifier":{"value":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}},"organization":{"identifier":{"value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":3,"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"430193006"}]},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":100.96600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":403.86400000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":504.83,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":504.83,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":516.2800000000001,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"90925926-2c5e-4592-bc67-e5a951052c94","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"2bf9a87f-6c7e-4f56-b63f-4a5675faefb7"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2011-03-11T13:30:05-08:00","end":"2012-03-11T13:30:05-07:00"},"created":"2011-03-11T13:30:05-08:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/2bf9a87f-6c7e-4f56-b63f-4a5675faefb7"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2011-03-11T13:15:05-08:00","end":"2011-03-11T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/ebe5ce5b-57bb-4368-93d6-0b7a0da073a1"}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"d457c5c5-bd89-48f6-9848-e703fd3b5b42","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"0dd76242-6c52-45ce-9e13-84e635c25f4a"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2011-06-09T14:30:05-07:00","end":"2012-06-09T14:30:05-07:00"},"provider":{"identifier":{"value":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}},"organization":{"identifier":{"value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"9a3c5807-b876-45c8-aba3-565f05dec4c8","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"7f84d563-6fb1-4339-8432-766672b1737b"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2013-02-28T13:30:05-08:00","end":"2014-02-28T13:30:05-08:00"},"created":"2013-02-28T13:30:05-08:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/7f84d563-6fb1-4339-8432-766672b1737b"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2013-02-28T13:15:05-08:00","end":"2013-02-28T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/6245224b-6d15-448a-8e56-db606acf21d8"}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"66a5b348-a9c5-40f3-8096-c20ace45c3bb","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"9e3e2f01-530d-40b1-9795-eba32a97158b"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2014-06-12T14:30:05-07:00","end":"2015-06-12T14:30:05-07:00"},"provider":{"identifier":{"value":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}},"organization":{"identifier":{"value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":3,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":4,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":5,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":449.66400000000004,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"1c795fd2-353b-4594-93de-af5b191f1b2b","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"0623afa7-5753-450b-9124-c93df6a8a4f7"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-02-18T13:30:05-08:00","end":"2016-02-18T13:30:05-08:00"},"created":"2015-02-18T13:30:05-08:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/0623afa7-5753-450b-9124-c93df6a8a4f7"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2015-02-18T13:15:05-08:00","end":"2015-02-18T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/7b2e1b32-79cc-412f-b8e7-97997da45089"}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"6a4aea36-e0fc-41b3-8d2d-202653cf2aec","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"abafc9ba-edc7-4eff-98fc-e41d8785c27b"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2015-04-12T14:41:05-07:00","end":"2016-04-12T14:41:05-07:00"},"provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-poa-ind-sw1-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-poa-ind-sw1","code":"Y","display":"Diagnosis present at time of admission"}}],"sequence":1,"diagnosisReference":{"reference":"Condition/5953e3df-ed6c-4509-8d56-04c745817bd0"},"type":[{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/diagnosis-type","code":"principal"}]}]}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":3,"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"399208008"}]},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":1307.116,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":5228.464,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":6535.58,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":6535.58,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":5228.464,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"88ae65ef-0f73-460c-a3d8-9a5dabb81440","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"9bd3bd28-4e88-4d8a-8510-ea791953455b"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2015-04-23T14:30:05-07:00","end":"2016-04-23T14:30:05-07:00"},"created":"2015-04-23T14:30:05-07:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/9bd3bd28-4e88-4d8a-8510-ea791953455b"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"}]},{"sequence":2,"informationLinkId":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]},"reason":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","code":"A","display":"Allowed"}]}}]},{"sequence":3,"informationLinkId":[2],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2015-04-23T14:15:05-07:00","end":"2015-04-23T14:30:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"net":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]},"reason":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","code":"A","display":"Allowed"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":224.83200000000002,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"198da8a5-4987-4df0-9be0-c74919811d87","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"36c0e892-2f4c-4387-8587-7fe004a6bafc"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-02-13T13:30:05-08:00","end":"2017-02-13T13:30:05-08:00"},"created":"2016-02-13T13:30:05-08:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/36c0e892-2f4c-4387-8587-7fe004a6bafc"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2016-02-13T13:15:05-08:00","end":"2016-02-13T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/96baca61-8639-457b-bd48-fa32511b5985"}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"5cc1e2c3-0bd6-4257-9a23-ba6629b034d0","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"d69ba5dd-6da7-4d7b-8cce-63b1ed0c5134"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2016-10-30T16:45:05-07:00","end":"2017-10-30T16:45:05-07:00"},"created":"2016-10-30T16:45:05-07:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/d69ba5dd-6da7-4d7b-8cce-63b1ed0c5134"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-poa-ind-sw1-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-poa-ind-sw1","code":"Y","display":"Diagnosis present at time of admission"}}],"sequence":1,"diagnosisReference":{"reference":"Condition/0aa16b8a-bce6-4362-8312-879d7ccf29f8"},"type":[{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/diagnosis-type","code":"principal"}]}]}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"23","display":"Emergency Room"}]},"encounter":[{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"}]},{"sequence":2,"diagnosisLinkId":[1],"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"23","display":"Emergency Room"}]}},{"sequence":3,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"168594001"}]},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"23","display":"Emergency Room"}]},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":103.33,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":413.32,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]},"reason":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","code":"A","display":"Allowed"}]}}]},{"sequence":4,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"305428000"}]},"servicedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T16:45:05-07:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"23","display":"Emergency Room"}]},"net":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":103.33,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":413.32,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":516.65,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]},"reason":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","code":"A","display":"Allowed"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":826.64,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"40a1df7e-d7fb-446b-8a76-99c8b3083cdc","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"4cb0502f-3892-41f6-ac59-49d31dc28193"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2016-12-29T13:30:05-08:00","end":"2017-12-29T13:30:05-08:00"},"provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"6428ae9b-f547-4924-8a74-40eccc8838b7","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"f7183ce1-f163-40b3-8ed8-0bcff268937f"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2017-06-15T14:45:05-07:00","end":"2018-06-15T14:45:05-07:00"},"provider":{"identifier":{"value":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8"}},"organization":{"identifier":{"value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":28.104000000000003,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":112.41600000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":140.52,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":3,"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"430193006"}]},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"22","display":"Outpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":96.404,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":385.616,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":482.02,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":482.02,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":498.032,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"722adc6a-fc27-4348-8d93-bd12b951d1f0","contained":[{"resourceType":"ReferralRequest","id":"1","status":"completed","intent":"order","subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"requester":{"agent":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}},"recipient":[{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}}]},{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"e72892e7-dbe8-45ce-b6bf-4d0952c618e3"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"start":"2018-02-02T13:30:05-08:00","end":"2019-02-02T13:30:05-08:00"},"created":"2018-02-02T13:30:05-08:00","provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"referral":{"reference":"#1"},"claim":{"reference":"Claim/e72892e7-dbe8-45ce-b6bf-4d0952c618e3"},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"sequence":1,"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_cms_type_srvc_cd","code":"1","display":"Medical care"}]},"servicedPeriod":{"start":"2018-02-02T13:15:05-08:00","end":"2018-02-02T13:30:05-08:00"},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"encounter":[{"reference":"Encounter/6fb5a1fb-c93c-4648-bb60-db870634bbf2"}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"73863e2b-5854-4742-9106-e95d3ea46813","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-ime-op-clm-val-amt-extension","valueMoney":{"value":400.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-dsh-op-clm-val-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pass-thru-per-diem-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-tot-pps-cptl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-bene-ip-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-bene-pta-coinsrnc-lblty-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-ip-ncvrd-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-ip-tot-ddctn-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-cptl-dsprprtnt-shr-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-cptl-excptn-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-cptl-fsp-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-cptl-ime-amt-extension","valueMoney":{"value":400.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-cptl-outlier-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-pps-old-cptl-hld-hrmls-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-drg-outlier-aprvd-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"8d5d5189-0c2c-4bf9-930a-7bc245908702"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2018-02-10T15:15:05-08:00","end":"2019-02-10T15:15:05-08:00"},"provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"service":{"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"287664005"}]},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":1872.8980000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":7491.592000000001,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":9364.49,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":9364.49,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":7491.592000000001,"system":"urn:iso:std:iso:4217","code":"USD"}}} +{"resourceType":"ExplanationOfBenefit","id":"e030c902-14f9-4fdc-8268-b8caac5c378f","meta":{"profile":["https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim"]},"contained":[{"resourceType":"Coverage","id":"coverage","type":{"text":"Medicaid"}}],"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-profnl-cmpnt-chrg-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-ddctbl-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-ptb-coinsrnc-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-prvdr-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-op-bene-pmt-amt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-nch-bene-blood-ddctbl-lblty-am-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-clm-mdcr-non-pmt-rsn-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-mdcr-non-pmt-rsn-cd","code":"N","display":"All other reasons for non-payment"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-prpayamt-extension","valueMoney":{"value":0.0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-fi-num-extension","valueIdentifier":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-fi-num","value":"002000"}}],"identifier":[{"system":"https://bluebutton.cms.gov/resources/variables/clm_id","value":"d220ef38-fff4-45e8-94e0-da6716f84ff6"},{"system":"https://bluebutton.cms.gov/resources/identifier/claim-group","value":"99999999999"}],"status":"active","type":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/nch_clm_type_cd","code":"71","display":"Local carrier non-durable medical equipment, prosthetics, orthotics, and supplies (DMEPOS) claim"},{"system":"https://bluebutton.cms.gov/resources/codesystem/eob-type","code":"CARRIER","display":"EOB Type"},{"system":"http://hl7.org/fhir/ex-claimtype","code":"professional","display":"Claim Type"},{"system":"https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd","code":"O","display":"Part B physician/supplier claim record (processed by local carriers; can include DMEPOS services)"}]},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"billablePeriod":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-claim-query-cd-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/ValueSet-claim-query-cd","code":"3","display":"Final Bill"}}],"start":"2018-12-11T13:45:05-08:00","end":"2019-12-11T13:45:05-08:00"},"provider":{"identifier":{"value":"c16820ae-2954-32d4-863c-e9ceb741154c"}},"organization":{"identifier":{"value":"465de31f-3098-365c-af70-48a071e1f5aa"}},"careTeam":[{"sequence":1,"provider":{"identifier":{"system":"http://hl7.org/fhir/sid/us-npi","value":"99999999"}},"role":{"coding":[{"system":"http://hl7.org/fhir/claimcareteamrole","code":"primary","display":"Primary Care Practitioner"}]}}],"diagnosis":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-inpatient-clm-poa-ind-sw1-extension","valueCoding":{"system":"https://bluebutton.cms.gov/assets/ig/CodeSystem-clm-poa-ind-sw1","code":"Y","display":"Diagnosis present at time of admission"}}],"sequence":1,"diagnosisReference":{"reference":"Condition/53dac70f-c5cc-4880-a96e-fdb157db8e3b"},"type":[{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/diagnosis-type","code":"principal"}]}]}],"insurance":{"coverage":{"reference":"#coverage"}},"item":[{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":1,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":2,"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]}},{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ndc-qty-extension","valueQuantity":{"value":0}}],"sequence":3,"service":{"extension":[{"url":"https://bluebutton.cms.gov/assets/ig/StructureDefinition-bluebutton-outpatient-rev-cntr-ide-ndc-upc-num-extension","valueCoding":{"system":"https://www.accessdata.fda.gov/scripts/cder/ndc","code":"0624","display":"Dummy"}}],"coding":[{"system":"http://snomed.info/sct","version":"v1","code":"117015009"}]},"locationCodeableConcept":{"coding":[{"system":"https://bluebutton.cms.gov/resources/variables/line_place_of_srvc_cd","code":"21","display":"Inpatient Hospital"}]},"adjudication":[{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_coinsrnc_amt","display":"Line Beneficiary Coinsurance Amount"}]},"amount":{"value":391.722,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prvdr_pmt_amt","display":"Line Provider Payment Amount"}]},"amount":{"value":1566.888,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_sbmtd_chrg_amt","display":"Line Submitted Charge Amount"}]},"amount":{"value":1958.61,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_alowd_chrg_amt","display":"Line Allowed Charge Amount"}]},"amount":{"value":1958.61,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_bene_ptb_ddctbl_amt","display":"Line Beneficiary Part B Deductible Amount"}]},"amount":{"value":0,"system":"urn:iso:std:iso:4217","code":"USD"}},{"category":{"coding":[{"system":"https://bluebutton.cms.gov/resources/codesystem/adjudication","code":"https://bluebutton.cms.gov/resources/variables/line_prcsg_ind_cd","display":"Line Processing Indicator Code"}]}}]}],"totalCost":{"value":129.16,"system":"urn:iso:std:iso:4217","code":"USD"},"payment":{"amount":{"value":1566.888,"system":"urn:iso:std:iso:4217","code":"USD"}}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ImagingStudy.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ImagingStudy.ndjson new file mode 100644 index 000000000000..9e226f86482e --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/ImagingStudy.ndjson @@ -0,0 +1 @@ +{"resourceType":"ImagingStudy","id":"992ab51b-103c-4d81-8582-6b54a83342ea","uid":"urn:oid:1.2.840.99999999.22502730.1589831196459","patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"started":"2016-10-30T14:15:05-07:00","numberOfSeries":1,"numberOfInstances":1,"series":[{"uid":"urn:oid:1.2.840.99999999.1.65971777.1589831196459","number":1,"modality":{"system":"http://dicom.nema.org/resources/ontology/DCM","code":"DX","display":"Digital Radiography"},"numberOfInstances":1,"availability":"UNAVAILABLE","bodySite":{"system":"http://snomed.info/sct","code":"51299004","display":"Clavicle"},"started":"2016-10-30T14:15:05-07:00","instance":[{"uid":"urn:oid:1.2.840.99999999.1.1.48010377.1589831196459","number":1,"sopClass":"urn:oid:1.2.840.10008.5.1.4.1.1.1.1","title":"Image of clavicle"}]}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Immunization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Immunization.ndjson new file mode 100644 index 000000000000..03e078d441ce --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Immunization.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"Immunization","id":"293b214e-ae9b-4053-8ed1-becbe613de18","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"date":"2010-06-03T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"c04ae3a8-1678-404b-8fb7-2d249ba82839","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"date":"2011-06-09T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"60ab5005-b572-498c-9e32-a5acdc26e3f8","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"date":"2014-06-12T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"dc26c48b-cdd3-4384-a0f2-02d25682a02d","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"113","display":"Td (adult) preservative free"}],"text":"Td (adult) preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"date":"2014-06-12T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"52ebf7c5-8af2-4f91-9e9a-8b54de12f857","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"date":"2014-06-12T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"ded2b0fb-56ed-4245-85c8-0b2a02ff4b53","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"114","display":"meningococcal MCV4P"}],"text":"meningococcal MCV4P"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"date":"2014-06-12T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"8ea94a08-14b2-4b18-9f70-7d5f8ae729f7","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"date":"2015-04-23T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"8ca4d799-58a3-4930-a35c-c6f5a1555173","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"43","display":"Hep B, adult"}],"text":"Hep B, adult"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"date":"2015-04-23T14:15:05-07:00","primarySource":true} +{"resourceType":"Immunization","id":"69152acf-13e8-4265-bd5f-7a2df80a08f2","status":"completed","notGiven":false,"vaccineCode":{"coding":[{"system":"http://hl7.org/fhir/sid/cvx","code":"140","display":"Influenza, seasonal, injectable, preservative free"}],"text":"Influenza, seasonal, injectable, preservative free"},"patient":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"encounter":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"date":"2017-06-15T14:15:05-07:00","primarySource":true} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/MedicationRequest.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/MedicationRequest.ndjson new file mode 100644 index 000000000000..718327c2cfcc --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/MedicationRequest.ndjson @@ -0,0 +1,9 @@ +{"resourceType":"MedicationRequest","id":"df4ce22a-1ae5-429f-8d76-5a503a3921ab","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/7b569404-f475-45d9-94d3-b4e31fece687"},"authoredOn":"2010-05-19T14:15:05-07:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}},"reasonReference":[{"reference":"Condition/f8fe7590-249c-4b93-8bf0-275f2feb6b81"}]} +{"resourceType":"MedicationRequest","id":"8c7b34ec-760b-412b-bfd4-ec560b2ac998","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"751905","display":"Trinessa 28 Day Pack"}],"text":"Trinessa 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/ebe5ce5b-57bb-4368-93d6-0b7a0da073a1"},"authoredOn":"2011-03-11T13:15:05-08:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}}} +{"resourceType":"MedicationRequest","id":"546161c7-df48-41ae-a25f-2dbbf39729be","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"748856","display":"Yaz 28 Day Pack"}],"text":"Yaz 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/6245224b-6d15-448a-8e56-db606acf21d8"},"authoredOn":"2013-02-28T13:15:05-08:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}}} +{"resourceType":"MedicationRequest","id":"c9970c0c-236e-4431-bfe9-c08cdd622411","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"1534809","display":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"}],"text":"168 HR Ethinyl Estradiol 0.00146 MG/HR / norelgestromin 0.00625 MG/HR Transdermal System"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/7b2e1b32-79cc-412f-b8e7-97997da45089"},"authoredOn":"2015-02-18T13:15:05-08:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}}} +{"resourceType":"MedicationRequest","id":"6cb4c447-aa9d-4bfc-89f3-e424dab1fd2a","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"313782","display":"Acetaminophen 325 MG Oral Tablet"}],"text":"Acetaminophen 325 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"},"authoredOn":"2015-04-12T14:15:05-07:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}},"reasonReference":[{"reference":"Condition/5953e3df-ed6c-4509-8d56-04c745817bd0"}]} +{"resourceType":"MedicationRequest","id":"e42328cf-bb27-470f-9cff-a9dcb8fd7485","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"978950","display":"Natazia 28 Day Pack"}],"text":"Natazia 28 Day Pack"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/96baca61-8639-457b-bd48-fa32511b5985"},"authoredOn":"2016-02-13T13:15:05-08:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}}} +{"resourceType":"MedicationRequest","id":"b8c23eb0-d9b6-4413-8293-62c13b47e2ab","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"861467","display":"Meperidine Hydrochloride 50 MG Oral Tablet"}],"text":"Meperidine Hydrochloride 50 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"authoredOn":"2016-10-30T14:15:05-07:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}},"dosageInstruction":[{"sequence":1,"timing":{"repeat":{"frequency":1,"period":4.0,"periodUnit":"h"}},"asNeededBoolean":false,"doseQuantity":{"value":1.0}}]} +{"resourceType":"MedicationRequest","id":"22ba50dd-b723-4771-9247-2b25e566da68","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"310965","display":"Ibuprofen 200 MG Oral Tablet"}],"text":"Ibuprofen 200 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"authoredOn":"2016-10-30T14:15:05-07:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}},"dosageInstruction":[{"sequence":1,"asNeededBoolean":true}]} +{"resourceType":"MedicationRequest","id":"f313632b-f3da-4507-a3de-954c156bd799","status":"stopped","intent":"order","medicationCodeableConcept":{"coding":[{"system":"http://www.nlm.nih.gov/research/umls/rxnorm","code":"834102","display":"Penicillin V Potassium 500 MG Oral Tablet"}],"text":"Penicillin V Potassium 500 MG Oral Tablet"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"},"authoredOn":"2018-12-11T13:15:05-08:00","requester":{"agent":{"reference":"Practitioner/c16820ae-2954-32d4-863c-e9ceb741154c"},"onBehalfOf":{"reference":"Organization/465de31f-3098-365c-af70-48a071e1f5aa"}},"reasonReference":[{"reference":"Condition/f8fe7590-249c-4b93-8bf0-275f2feb6b81"},{"reference":"Condition/53dac70f-c5cc-4880-a96e-fdb157db8e3b"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Observation.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Observation.ndjson new file mode 100644 index 000000000000..56e2b92c6649 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Observation.ndjson @@ -0,0 +1,54 @@ +{"resourceType":"Observation","id":"1bd4f633-9348-41ae-aaa9-7870c0e2c840","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":153.6,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"a42f60ca-c6c0-48e2-963a-17d022452b98","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":1,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"df7d1c32-d860-45a3-bda4-32f32d866a32","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":55.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"29c6ee5d-8755-449d-b9a2-dd15c74d225b","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":23.41,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"839d525b-6730-42e2-b87f-1a4da2afab5f","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":71.788,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"c2505847-c2b3-4cdd-9042-527fa5413427","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":72,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":120,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"71a18dd9-86bb-4592-98a0-c5f1a1a8432a","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"7b451550-d8e1-4876-a7c4-dcb2b239ed77","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"3e63fc53-2b1d-4aa8-a258-4c6f0b913426","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"effectiveDateTime":"2010-06-03T14:15:05-07:00","issued":"2010-06-03T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"34434c43-6a31-4338-812f-2f8b9f3fe6d1","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":153.7,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"694f4732-4f02-4885-a62d-49a569cca6a7","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"894cddeb-1759-4714-a7db-16c2ecad0be7","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":62.1,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"0e9e67fc-c729-479e-9284-56399ccfb04b","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":26.3,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"46d8334e-7631-4a4e-936b-b451fdf795f9","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"59576-9","display":"Body mass index (BMI) [Percentile] Per age and gender"}],"text":"Body mass index (BMI) [Percentile] Per age and gender"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":85.491,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"2340483d-6118-4dfd-9256-37b815fc623d","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":88,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":121,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"21ccc32a-5d37-4dc7-b1bf-aefd6c930708","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":93,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"d6c19b7f-3e0b-4841-8ce1-3cbd7f7459f8","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueQuantity":{"value":13,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"c8ba5ba3-1bcd-4043-aade-0a9b109724b5","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/3a99bba3-f304-4d33-826a-018499a08f5a"},"effectiveDateTime":"2011-06-09T14:15:05-07:00","issued":"2011-06-09T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"bde88c9f-40fa-44d0-a465-05523c0f25c6","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"f3ec4455-433a-468a-984f-ddd1f1edef66","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"4e4efc47-10c1-43d7-9481-a7424a2b06e0","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":69.2,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"77fd7fe3-ae38-4ec4-a18d-a7e24ed97fc7","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":29.25,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"76bac1a4-fe2a-472c-af3e-f5c1234eef5a","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":87,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":122,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"af1fa8f5-efef-4b15-959d-6a66f9a5bf16","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":90,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"ed5a9f51-3e45-44d1-8208-604cac028376","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":16,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"a71330cb-bdf0-4383-9411-0e3372da9577","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"6690-2","display":"Leukocytes [#/volume] in Blood by Automated count"}],"text":"Leukocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":7.2857,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"f74b75ea-1ad6-4418-ae8b-2bd62e875e21","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"789-8","display":"Erythrocytes [#/volume] in Blood by Automated count"}],"text":"Erythrocytes [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":4.7109,"unit":"10*6/uL","system":"http://unitsofmeasure.org","code":"10*6/uL"}} +{"resourceType":"Observation","id":"573fb355-b0bf-4af5-98d8-ead5389eb3cc","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"718-7","display":"Hemoglobin [Mass/volume] in Blood"}],"text":"Hemoglobin [Mass/volume] in Blood"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":15.964,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"d750ec32-b61a-40da-b58d-fff11de5bf27","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"4544-3","display":"Hematocrit [Volume Fraction] of Blood by Automated count"}],"text":"Hematocrit [Volume Fraction] of Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":45.934,"unit":"%","system":"http://unitsofmeasure.org","code":"%"}} +{"resourceType":"Observation","id":"ab86a392-7cca-4625-97d4-421aeaa14382","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"787-2","display":"MCV [Entitic volume] by Automated count"}],"text":"MCV [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":94.793,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"0bb6c8de-bf30-43bd-9ae7-b56bc3c74914","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"785-6","display":"MCH [Entitic mass] by Automated count"}],"text":"MCH [Entitic mass] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":28.937,"unit":"pg","system":"http://unitsofmeasure.org","code":"pg"}} +{"resourceType":"Observation","id":"ff40e33c-caee-4181-b71f-cf9a12f7a24f","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"786-4","display":"MCHC [Mass/volume] by Automated count"}],"text":"MCHC [Mass/volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":33.479,"unit":"g/dL","system":"http://unitsofmeasure.org","code":"g/dL"}} +{"resourceType":"Observation","id":"2e6accca-09d2-462d-b7dc-777dfa52d06b","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"21000-5","display":"Erythrocyte distribution width [Entitic volume] by Automated count"}],"text":"Erythrocyte distribution width [Entitic volume] by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":43.303,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"0ac11784-c312-44c4-9942-b7d2beb26056","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"777-3","display":"Platelets [#/volume] in Blood by Automated count"}],"text":"Platelets [#/volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":173.89,"unit":"10*3/uL","system":"http://unitsofmeasure.org","code":"10*3/uL"}} +{"resourceType":"Observation","id":"d29445ad-c8cf-4ced-b59a-a4683c04ce51","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"32207-3","display":"Platelet distribution width [Entitic volume] in Blood by Automated count"}],"text":"Platelet distribution width [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":413.95,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"c80a8ad6-4bf5-476e-b3ef-09d812e14ded","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"laboratory","display":"laboratory"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"32623-1","display":"Platelet mean volume [Entitic volume] in Blood by Automated count"}],"text":"Platelet mean volume [Entitic volume] in Blood by Automated count"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueQuantity":{"value":10.07,"unit":"fL","system":"http://unitsofmeasure.org","code":"fL"}} +{"resourceType":"Observation","id":"99f42a3c-5eb5-49db-8448-b24cfc67b24d","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/4ef1ef04-c4f7-4739-b722-e668ec9f6c35"},"effectiveDateTime":"2014-06-12T14:15:05-07:00","issued":"2014-06-12T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"7c5bd16f-53b7-4d34-8c81-c0ec8b52bbe1","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"5e71e502-49b6-4aa4-8fb1-a1b8b9a9b16c","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":2,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"cdd7d477-1300-4b05-b401-f496a6a5b6fd","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":70.3,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"b3b07e6c-6406-49e2-b36d-8df17c3beed0","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":29.73,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"75ffe915-fb72-49ad-a8ca-3a775a2c1bdd","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":79,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":129,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"6068a754-23b6-4b25-b2cb-a445ef6a00ad","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":90,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"036fb690-7a46-43d0-9b10-bea7e5471816","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueQuantity":{"value":15,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"e9e4c78f-5076-451e-a1f9-a3bf65ceea48","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/74003abb-cee0-4151-8469-4a815fff57aa"},"effectiveDateTime":"2015-04-23T14:15:05-07:00","issued":"2015-04-23T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"75c27e1b-e4b4-4273-bf81-ac672aec77ff","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8302-2","display":"Body Height"}],"text":"Body Height"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":153.8,"unit":"cm","system":"http://unitsofmeasure.org","code":"cm"}} +{"resourceType":"Observation","id":"656b0220-e028-48b0-8ce2-3b9e63c0940a","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72514-3","display":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"}],"text":"Pain severity - 0-10 verbal numeric rating [Score] - Reported"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":0,"unit":"{score}","system":"http://unitsofmeasure.org","code":"{score}"}} +{"resourceType":"Observation","id":"dda3a68b-c604-4ac6-b033-aef7f3dfc5a4","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"29463-7","display":"Body Weight"}],"text":"Body Weight"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66.8,"unit":"kg","system":"http://unitsofmeasure.org","code":"kg"}} +{"resourceType":"Observation","id":"9eed66ed-362b-48bd-b464-b089990ad19a","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"39156-5","display":"Body Mass Index"}],"text":"Body Mass Index"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":28.22,"unit":"kg/m2","system":"http://unitsofmeasure.org","code":"kg/m2"}} +{"resourceType":"Observation","id":"26c30c4b-0461-4b6f-a84f-e6b075438799","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"85354-9","display":"Blood Pressure"}],"text":"Blood Pressure"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8462-4","display":"Diastolic Blood Pressure"}],"text":"Diastolic Blood Pressure"},"valueQuantity":{"value":69,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}},{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","display":"Systolic Blood Pressure"}],"text":"Systolic Blood Pressure"},"valueQuantity":{"value":111,"unit":"mm[Hg]","system":"http://unitsofmeasure.org","code":"mm[Hg]"}}]} +{"resourceType":"Observation","id":"3581b2d3-4c7a-46f0-81c5-33f249738522","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}],"text":"Heart rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":66,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"1258d6ee-aed7-48ac-b8a5-058940f9d48b","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"9279-1","display":"Respiratory rate"}],"text":"Respiratory rate"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueQuantity":{"value":12,"unit":"/min","system":"http://unitsofmeasure.org","code":"/min"}} +{"resourceType":"Observation","id":"7339a567-e371-4a13-8fb9-b6c2a61938ac","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"survey","display":"survey"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"72166-2","display":"Tobacco smoking status NHIS"}],"text":"Tobacco smoking status NHIS"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"effectiveDateTime":"2017-06-15T14:15:05-07:00","issued":"2017-06-15T14:15:05.225-07:00","valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"266919005","display":"Never smoker"}],"text":"Never smoker"}} +{"resourceType":"Observation","id":"1902e0c7-6e2b-42b0-9f37-65e94511286d","status":"final","category":[{"coding":[{"system":"http://hl7.org/fhir/observation-category","code":"vital-signs","display":"vital-signs"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"8310-5","display":"Body temperature"}],"text":"Body temperature"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"},"effectiveDateTime":"2018-12-11T13:15:05-08:00","issued":"2018-12-11T13:15:05.225-08:00","valueQuantity":{"value":39.024,"unit":"Cel","system":"http://unitsofmeasure.org","code":"Cel"}} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Organization.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Organization.ndjson new file mode 100644 index 000000000000..06c5160f19af --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Organization.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Organization","id":"465de31f-3098-365c-af70-48a071e1f5aa","identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"465de31f-3098-365c-af70-48a071e1f5aa"}],"type":[{"coding":[{"system":"http://hl7.org/fhir/organization-type","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"}],"name":"METROWEST MEDICAL CENTER","telecom":[{"system":"phone","value":"5083831000"}],"address":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/geolocation","extension":[{"url":"latitude","valueDecimal":42.307905},{"url":"longitude","valueDecimal":-71.436196}]}],"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}]} +{"resourceType":"Organization","id":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8","identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"58fe1815-1e8a-38ed-a91a-17d4ef18c8d8"}],"type":[{"coding":[{"system":"http://hl7.org/fhir/organization-type","code":"prov","display":"Healthcare Provider"}],"text":"Healthcare Provider"}],"name":"PCP68975","telecom":[{"system":"phone","value":"508-881-4368"}],"address":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/geolocation","extension":[{"url":"latitude","valueDecimal":42.257754999999996},{"url":"longitude","valueDecimal":-71.473526}]}],"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Patient.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Patient.ndjson new file mode 100644 index 000000000000..3243439fb2f8 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Patient.ndjson @@ -0,0 +1 @@ +{"resourceType":"Patient","id":"1416dec1-f4b1-4b48-b7f4-650e8f67499c","text":{"status":"generated","div":"
Generated by Synthea.Version identifier: v2.5.0-265-gbd5a00e8\n . Person seed: 6732543839779682504 Population seed: 1589831189867
"},"extension":[{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-race","extension":[{"url":"ombCategory","valueCoding":{"system":"urn:oid:2.16.840.1.113883.6.238","code":"2106-3","display":"White"}},{"url":"text","valueString":"White"}]},{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity","extension":[{"url":"ombCategory","valueCoding":{"system":"urn:oid:2.16.840.1.113883.6.238","code":"2186-5","display":"Not Hispanic or Latino"}},{"url":"text","valueString":"Not Hispanic or Latino"}]},{"url":"http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName","valueString":"Leanna255 Predovic534"},{"url":"http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex","valueCode":"F"},{"url":"http://hl7.org/fhir/StructureDefinition/birthPlace","valueAddress":{"city":"Southbridge","state":"Massachusetts","country":"US"}},{"url":"http://synthetichealth.github.io/synthea/disability-adjusted-life-years","valueDecimal":0.0},{"url":"http://synthetichealth.github.io/synthea/quality-adjusted-life-years","valueDecimal":27.0}],"identifier":[{"system":"https://github.com/synthetichealth/synthea","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://hl7.org/fhir/v2/0203","code":"MR","display":"Medical Record Number"}],"text":"Medical Record Number"},"system":"http://hospital.smarthealthit.org","value":"1416dec1-f4b1-4b48-b7f4-650e8f67499c"},{"type":{"coding":[{"system":"http://hl7.org/fhir/identifier-type","code":"SB","display":"Social Security Number"}],"text":"Social Security Number"},"system":"http://hl7.org/fhir/sid/us-ssn","value":"999-54-3579"},{"type":{"coding":[{"system":"http://hl7.org/fhir/v2/0203","code":"DL","display":"Driver's License"}],"text":"Driver's License"},"system":"urn:oid:2.16.840.1.113883.4.3.25","value":"S99972984"},{"type":{"coding":[{"system":"http://hl7.org/fhir/v2/0203","code":"PPN","display":"Passport Number"}],"text":"Passport Number"},"system":"http://standardhealthrecord.org/fhir/StructureDefinition/passportNumber","value":"X39621585X"}],"name":[{"use":"official","family":"Nolan344","given":["Lorita217"],"prefix":["Ms."]}],"telecom":[{"system":"phone","value":"555-817-6998","use":"home"}],"gender":"female","birthDate":"1992-04-09","address":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/geolocation","extension":[{"url":"latitude","valueDecimal":42.27693107900605},{"url":"longitude","valueDecimal":-71.45741653702677}]}],"line":["330 Sawayn Parade"],"city":"Framingham","state":"Massachusetts","country":"US"}],"maritalStatus":{"coding":[{"system":"http://hl7.org/fhir/v3/MaritalStatus","code":"S","display":"S"}],"text":"S"},"multipleBirthBoolean":false,"communication":[{"language":{"coding":[{"system":"urn:ietf:bcp:47","code":"en-US","display":"English"}],"text":"English"}}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Practitioner.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Practitioner.ndjson new file mode 100644 index 000000000000..eba7b8c26455 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Practitioner.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Practitioner","id":"c16820ae-2954-32d4-863c-e9ceb741154c","identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"530"}],"active":true,"name":[{"family":"Murphy561","given":["Mari763"],"prefix":["Dr."]}],"address":[{"line":["115 LINCOLN STREET"],"city":"FRAMINGHAM","state":"MA","postalCode":"01701","country":"US"}],"gender":"female"} +{"resourceType":"Practitioner","id":"a0f721c3-b8d1-3227-a8bf-27e5bf1ef5e8","identifier":[{"system":"http://hl7.org/fhir/sid/us-npi","value":"35750"}],"active":true,"name":[{"family":"Hilpert278","given":["Cathryn51"],"prefix":["Dr."]}],"address":[{"line":["259 MAIN ST"],"city":"ASHLAND","state":"MA","postalCode":"01721-2115","country":"US"}],"gender":"female"} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Procedure.ndjson b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Procedure.ndjson new file mode 100644 index 000000000000..3a484c87be00 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/resources/resources/STU3/Procedure.ndjson @@ -0,0 +1,7 @@ +{"resourceType":"Procedure","id":"63702bbb-e289-4a80-9f1a-3ef5a7d15019","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/d122a0a0-c565-4e44-a83f-6b9ddd7f8cd7"},"performedPeriod":{"start":"2010-06-03T14:15:05-07:00","end":"2010-06-03T14:30:05-07:00"}} +{"resourceType":"Procedure","id":"bbedb163-5b9d-4865-bd9f-d59f7a9c21ad","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"399208008","display":"Plain chest X-ray (procedure)"}],"text":"Plain chest X-ray (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/37cce86a-e35b-4879-b8ed-4f22b10b2f7c"},"performedPeriod":{"start":"2015-04-12T14:15:05-07:00","end":"2015-04-12T14:26:05-07:00"},"reasonReference":[{"reference":"Condition/5953e3df-ed6c-4509-8d56-04c745817bd0","display":"Acute bronchitis (disorder)"}]} +{"resourceType":"Procedure","id":"565fedff-5363-4938-9b49-1a2b5b004e15","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"168594001","display":"Clavicle X-ray"}],"text":"Clavicle X-ray"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T14:45:05-07:00"}} +{"resourceType":"Procedure","id":"faae5e23-a03a-450a-ad12-50ca56eaa0e1","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"305428000","display":"Admission to orthopedic department"}],"text":"Admission to orthopedic department"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/187a1ecd-b5f5-4566-b39d-ae646062cfbc"},"performedPeriod":{"start":"2016-10-30T14:15:05-07:00","end":"2016-10-30T15:15:05-07:00"},"reasonReference":[{"reference":"Condition/0aa16b8a-bce6-4362-8312-879d7ccf29f8","display":"Fracture of clavicle"}]} +{"resourceType":"Procedure","id":"2b4a074e-e62c-4208-9ae5-be371b9208d3","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"430193006","display":"Medication Reconciliation (procedure)"}],"text":"Medication Reconciliation (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/fc81047d-ee3a-4857-b96c-0a091e957d7a"},"performedPeriod":{"start":"2017-06-15T14:15:05-07:00","end":"2017-06-15T14:30:05-07:00"}} +{"resourceType":"Procedure","id":"28e04a1a-c56a-4aca-a144-ec986f09f1aa","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"287664005","display":"Bilateral tubal ligation"}],"text":"Bilateral tubal ligation"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/89c75588-d3a0-4ff8-8202-665ff4349fa7"},"performedPeriod":{"start":"2018-02-09T13:15:05-08:00","end":"2018-02-09T15:15:05-08:00"}} +{"resourceType":"Procedure","id":"17543a9f-3947-4a3c-af28-21391ee2674a","status":"completed","code":{"coding":[{"system":"http://snomed.info/sct","code":"117015009","display":"Throat culture (procedure)"}],"text":"Throat culture (procedure)"},"subject":{"reference":"Patient/1416dec1-f4b1-4b48-b7f4-650e8f67499c"},"context":{"reference":"Encounter/99ff4b9b-41f5-41ce-ac7c-b96808cc7ed9"},"performedPeriod":{"start":"2018-12-11T13:15:05-08:00","end":"2018-12-11T13:30:05-08:00"},"reasonReference":[{"reference":"Condition/f8fe7590-249c-4b93-8bf0-275f2feb6b81","display":"Streptococcal sore throat (disorder)"},{"reference":"Condition/53dac70f-c5cc-4880-a96e-fdb157db8e3b","display":"Streptococcal sore throat (disorder)"}]} diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Amelia635_Krajcik437_ed7a9f5c-37ea-4767-95ba-a6783e9500b3.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Amelia635_Krajcik437_ed7a9f5c-37ea-4767-95ba-a6783e9500b3.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Amelia635_Krajcik437_ed7a9f5c-37ea-4767-95ba-a6783e9500b3.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Amelia635_Krajcik437_ed7a9f5c-37ea-4767-95ba-a6783e9500b3.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Elma843_Hoppe518_1fd2683f-2a56-47c7-b674-98f2cc8319e7.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Elma843_Hoppe518_1fd2683f-2a56-47c7-b674-98f2cc8319e7.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Elma843_Hoppe518_1fd2683f-2a56-47c7-b674-98f2cc8319e7.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Elma843_Hoppe518_1fd2683f-2a56-47c7-b674-98f2cc8319e7.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Ernesto186_Dietrich576_4ecb4cbb-6df7-41e0-8e89-6e7a142721a5.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Ernesto186_Dietrich576_4ecb4cbb-6df7-41e0-8e89-6e7a142721a5.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Ernesto186_Dietrich576_4ecb4cbb-6df7-41e0-8e89-6e7a142721a5.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Ernesto186_Dietrich576_4ecb4cbb-6df7-41e0-8e89-6e7a142721a5.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Kortney212_Bosco882_8ed31d3e-7352-4055-89c4-f017db3f594e.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Kortney212_Bosco882_8ed31d3e-7352-4055-89c4-f017db3f594e.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Kortney212_Bosco882_8ed31d3e-7352-4055-89c4-f017db3f594e.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Kortney212_Bosco882_8ed31d3e-7352-4055-89c4-f017db3f594e.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Terry864_Hintz995_f04ef974-6d6f-4e1c-804f-cd4d62aabb4f.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Terry864_Hintz995_f04ef974-6d6f-4e1c-804f-cd4d62aabb4f.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/Terry864_Hintz995_f04ef974-6d6f-4e1c-804f-cd4d62aabb4f.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/Terry864_Hintz995_f04ef974-6d6f-4e1c-804f-cd4d62aabb4f.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/hospitalInformation1586309771387.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/hospitalInformation1586309771387.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/hospitalInformation1586309771387.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/hospitalInformation1586309771387.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/practitionerInformation1586309771387.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/practitionerInformation1586309771387.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/DSTU2/practitionerInformation1586309771387.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/DSTU2/practitionerInformation1586309771387.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/Emerson869_Prohaska837_d89e3bd8-4d0f-4735-8d16-d0f646a9dcd6.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Emerson869_Prohaska837_d89e3bd8-4d0f-4735-8d16-d0f646a9dcd6.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/Emerson869_Prohaska837_d89e3bd8-4d0f-4735-8d16-d0f646a9dcd6.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Emerson869_Prohaska837_d89e3bd8-4d0f-4735-8d16-d0f646a9dcd6.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/Lorette239_Marvin195_af0e4a42-a1ef-4c2f-ad31-9338a8b8fb7b.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Lorette239_Marvin195_af0e4a42-a1ef-4c2f-ad31-9338a8b8fb7b.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/Lorette239_Marvin195_af0e4a42-a1ef-4c2f-ad31-9338a8b8fb7b.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Lorette239_Marvin195_af0e4a42-a1ef-4c2f-ad31-9338a8b8fb7b.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/Norberto865_Cole117_a5e08ef9-fd71-4273-a53a-d5f5df7926f4.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Norberto865_Cole117_a5e08ef9-fd71-4273-a53a-d5f5df7926f4.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/Norberto865_Cole117_a5e08ef9-fd71-4273-a53a-d5f5df7926f4.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Norberto865_Cole117_a5e08ef9-fd71-4273-a53a-d5f5df7926f4.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/Robbyn526_DuBuque211_fcf2f472-77ac-47d7-9c9a-a6702bd2bb80.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Robbyn526_DuBuque211_fcf2f472-77ac-47d7-9c9a-a6702bd2bb80.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/Robbyn526_DuBuque211_fcf2f472-77ac-47d7-9c9a-a6702bd2bb80.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Robbyn526_DuBuque211_fcf2f472-77ac-47d7-9c9a-a6702bd2bb80.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/Seymour882_Shanahan202_55a5307d-5f23-49c1-9100-7d5c513abca3.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Seymour882_Shanahan202_55a5307d-5f23-49c1-9100-7d5c513abca3.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/Seymour882_Shanahan202_55a5307d-5f23-49c1-9100-7d5c513abca3.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/Seymour882_Shanahan202_55a5307d-5f23-49c1-9100-7d5c513abca3.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/hospitalInformation1586368892823.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/hospitalInformation1586368892823.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/hospitalInformation1586368892823.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/hospitalInformation1586368892823.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/R4/practitionerInformation1586368892823.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/practitionerInformation1586368892823.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/R4/practitionerInformation1586368892823.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/R4/practitionerInformation1586368892823.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Alexander630_Romaguera67_02a63c07-9fcc-42ba-aec0-9d5399ac4796.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Alexander630_Romaguera67_02a63c07-9fcc-42ba-aec0-9d5399ac4796.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Alexander630_Romaguera67_02a63c07-9fcc-42ba-aec0-9d5399ac4796.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Alexander630_Romaguera67_02a63c07-9fcc-42ba-aec0-9d5399ac4796.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Basil991_Pfannerstill264_88fb71f7-d445-4e5b-8af2-962e8f8e5fb6.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Basil991_Pfannerstill264_88fb71f7-d445-4e5b-8af2-962e8f8e5fb6.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Basil991_Pfannerstill264_88fb71f7-d445-4e5b-8af2-962e8f8e5fb6.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Basil991_Pfannerstill264_88fb71f7-d445-4e5b-8af2-962e8f8e5fb6.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Cris921_Lang846_df27a976-5c5e-4b84-ad00-fe32972dce9c.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Cris921_Lang846_df27a976-5c5e-4b84-ad00-fe32972dce9c.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Cris921_Lang846_df27a976-5c5e-4b84-ad00-fe32972dce9c.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Cris921_Lang846_df27a976-5c5e-4b84-ad00-fe32972dce9c.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Damon455_Will178_136f997d-0a94-4573-97da-a53b5060a612.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Damon455_Will178_136f997d-0a94-4573-97da-a53b5060a612.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Damon455_Will178_136f997d-0a94-4573-97da-a53b5060a612.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Damon455_Will178_136f997d-0a94-4573-97da-a53b5060a612.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Dannette613_Maggio310_74779846-85a4-4b26-9da4-414a5fec1aed.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Dannette613_Maggio310_74779846-85a4-4b26-9da4-414a5fec1aed.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Dannette613_Maggio310_74779846-85a4-4b26-9da4-414a5fec1aed.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Dannette613_Maggio310_74779846-85a4-4b26-9da4-414a5fec1aed.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/Shavonne800_Hilll811_3024090f-fe14-40a9-8fae-79952d3c95ce.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Shavonne800_Hilll811_3024090f-fe14-40a9-8fae-79952d3c95ce.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/Shavonne800_Hilll811_3024090f-fe14-40a9-8fae-79952d3c95ce.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/Shavonne800_Hilll811_3024090f-fe14-40a9-8fae-79952d3c95ce.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/hospitalInformation1586298239556.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/hospitalInformation1586298239556.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/hospitalInformation1586298239556.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/hospitalInformation1586298239556.json diff --git a/sdks/java/io/google-cloud-platform/src/test/resources/STU3/practitionerInformation1586298239556.json b/sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/practitionerInformation1586298239556.json similarity index 100% rename from sdks/java/io/google-cloud-platform/src/test/resources/STU3/practitionerInformation1586298239556.json rename to sdks/java/io/google-cloud-platform/src/test/resources/transactional_bundles/STU3/practitionerInformation1586298239556.json diff --git a/sdks/java/io/kudu/build.gradle b/sdks/java/io/kudu/build.gradle index 0090e13336e0..cda962d4d6ee 100644 --- a/sdks/java/io/kudu/build.gradle +++ b/sdks/java/io/kudu/build.gradle @@ -17,7 +17,7 @@ */ plugins { id 'org.apache.beam.module' } -applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.sdk.io.kudu') +applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.io.kudu') provideIntegrationTestingDependencies() enableJavaPerformanceTesting() diff --git a/sdks/java/io/rabbitmq/build.gradle b/sdks/java/io/rabbitmq/build.gradle index a5f1055cce95..83cf7c4591ab 100644 --- a/sdks/java/io/rabbitmq/build.gradle +++ b/sdks/java/io/rabbitmq/build.gradle @@ -17,7 +17,7 @@ */ plugins { id 'org.apache.beam.module' } -applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.sdk.io.rabbitmq') +applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.io.rabbitmq') description = "Apache Beam :: SDKs :: Java :: IO :: RabbitMQ" ext.summary = "IO to read and write to a RabbitMQ broker." diff --git a/sdks/java/io/snowflake/build.gradle b/sdks/java/io/snowflake/build.gradle new file mode 100644 index 000000000000..c51c0348efc0 --- /dev/null +++ b/sdks/java/io/snowflake/build.gradle @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { id 'org.apache.beam.module' } +applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.io.snowflake') +provideIntegrationTestingDependencies() +enableJavaPerformanceTesting() +description = "Apache Beam :: SDKs :: Java :: IO :: Snowflake" +ext.summary = "IO to read and write on Snowflake." +dependencies { + compile library.java.vendored_guava_26_0_jre + compile project(path: ":sdks:java:core", configuration: "shadow") + compile project(path: ":sdks:java:extensions:google-cloud-platform-core") + compile library.java.slf4j_api + compile group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.11.0' + compile group: 'com.opencsv', name: 'opencsv', version: '5.0' + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "testRuntime") + testCompile project(path: ":sdks:java:testing:test-utils", configuration: "testRuntime") + testCompile library.java.avro + testCompile library.java.junit + testCompile library.java.hamcrest_core + testCompile library.java.hamcrest_library + testCompile library.java.slf4j_api + testRuntimeOnly library.java.hadoop_client + testRuntimeOnly library.java.slf4j_jdk14 + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/CloudProvider.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/CloudProvider.java new file mode 100644 index 000000000000..404859c96d81 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/CloudProvider.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake; + +public enum CloudProvider { + GCS("gs://"); + + private final String prefix; + + private CloudProvider(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java new file mode 100644 index 000000000000..a67ba320d361 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java @@ -0,0 +1,759 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake; + +import static org.apache.beam.sdk.io.TextIO.readFiles; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import java.io.IOException; +import java.io.Serializable; +import java.security.PrivateKey; +import java.sql.Connection; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import javax.sql.DataSource; +import net.snowflake.client.jdbc.SnowflakeBasicDataSource; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.io.FileIO; +import org.apache.beam.sdk.io.FileSystems; +import org.apache.beam.sdk.io.fs.MoveOptions; +import org.apache.beam.sdk.io.fs.ResourceId; +import org.apache.beam.sdk.io.snowflake.credentials.KeyPairSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.OAuthTokenSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.SnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.UsernamePasswordSnowflakeCredentials; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Reshuffle; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.Wait; +import org.apache.beam.sdk.transforms.display.DisplayData; +import org.apache.beam.sdk.transforms.display.HasDisplayData; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * IO to read and write data on Snowflake. + * + *

SnowflakeIO uses Snowflake + * JDBC driver under the hood, but data isn't read/written using JDBC directly. Instead, + * SnowflakeIO uses dedicated COPY operations to read/write data from/to a cloud bucket. By + * now only Google Cloud Storage is supported. + * + *

To configure SnowflakeIO to read/write from your Snowflake instance, you have to provide a + * {@link DataSourceConfiguration} using {@link + * DataSourceConfiguration#create(SnowflakeCredentials)}, where {@link SnowflakeCredentials might be + * created using {@link org.apache.beam.sdk.io.snowflake.credentials.SnowflakeCredentialsFactory}}. + * Additionally one of {@link DataSourceConfiguration#withServerName(String)} or {@link + * DataSourceConfiguration#withUrl(String)} must be used to tell SnowflakeIO which instance to use. + *
+ * There are also other options available to configure connection to Snowflake: + * + *

    + *
  • {@link DataSourceConfiguration#withWarehouse(String)} to specify which Warehouse to use + *
  • {@link DataSourceConfiguration#withDatabase(String)} to specify which Database to connect + * to + *
  • {@link DataSourceConfiguration#withSchema(String)} to specify which schema to use + *
  • {@link DataSourceConfiguration#withRole(String)} to specify which role to use + *
  • {@link DataSourceConfiguration#withLoginTimeout(Integer)} to specify the timeout for the + * login + *
  • {@link DataSourceConfiguration#withPortNumber(Integer)} to specify custom port of Snowflake + * instance + *
+ * + *

For example: + * + *

{@code
+ * SnowflakeIO.DataSourceConfiguration dataSourceConfiguration =
+ *     SnowflakeIO.DataSourceConfiguration.create(SnowflakeCredentialsFactory.of(options))
+ *         .withServerName(options.getServerName())
+ *         .withWarehouse(options.getWarehouse())
+ *         .withDatabase(options.getDatabase())
+ *         .withSchema(options.getSchema());
+ * }
+ * + *

Reading from Snowflake

+ * + *

SnowflakeIO.Read returns a bounded collection of {@code T} as a {@code PCollection}. T is + * the type returned by the provided {@link CsvMapper}. + * + *

For example + * + *

{@code
+ * PCollection items = pipeline.apply(
+ *  SnowflakeIO.read()
+ *    .withDataSourceConfiguration(dataSourceConfiguration)
+ *    .fromQuery(QUERY)
+ *    .withStagingBucketName(stagingBucketName)
+ *    .withIntegrationName(integrationName)
+ *    .withCsvMapper(...)
+ *    .withCoder(...));
+ * }
+ * + *

Important When reading data from Snowflake, temporary CSV files are created on the + * specified stagingBucketName in directory named `sf_copy_csv_[RANDOM CHARS]_[TIMESTAMP]`. This + * directory and all the files are cleaned up automatically by default, but in case of failed + * pipeline they may remain and will have to be cleaned up manually. + */ +public class SnowflakeIO { + private static final Logger LOG = LoggerFactory.getLogger(SnowflakeIO.class); + + private static final String CSV_QUOTE_CHAR = "'"; + /** + * Read data from Snowflake. + * + * @param snowflakeService user-defined {@link SnowflakeService} + * @param Type of the data to be read. + */ + public static Read read(SnowflakeService snowflakeService) { + return new AutoValue_SnowflakeIO_Read.Builder() + .setSnowflakeService(snowflakeService) + .build(); + } + + /** + * Read data from Snowflake. + * + * @param Type of the data to be read. + */ + public static Read read() { + return read(new SnowflakeServiceImpl()); + } + + /** + * Interface for user-defined function mapping parts of CSV line into T. Used for + * SnowflakeIO.Read. + * + * @param Type of data to be read. + */ + @FunctionalInterface + public interface CsvMapper extends Serializable { + T mapRow(String[] parts) throws Exception; + } + + /** Implementation of {@link #read()}. */ + @AutoValue + public abstract static class Read extends PTransform> { + @Nullable + abstract SerializableFunction getDataSourceProviderFn(); + + @Nullable + abstract String getQuery(); + + @Nullable + abstract String getTable(); + + @Nullable + abstract String getIntegrationName(); + + @Nullable + abstract String getStagingBucketName(); + + @Nullable + abstract CsvMapper getCsvMapper(); + + @Nullable + abstract Coder getCoder(); + + @Nullable + abstract SnowflakeService getSnowflakeService(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setDataSourceProviderFn( + SerializableFunction dataSourceProviderFn); + + abstract Builder setQuery(String query); + + abstract Builder setTable(String table); + + abstract Builder setIntegrationName(String integrationName); + + abstract Builder setStagingBucketName(String stagingBucketName); + + abstract Builder setCsvMapper(CsvMapper csvMapper); + + abstract Builder setCoder(Coder coder); + + abstract Builder setSnowflakeService(SnowflakeService snowflakeService); + + abstract Read build(); + } + + /** + * Setting information about Snowflake server. + * + * @param config - An instance of {@link DataSourceConfiguration}. + */ + public Read withDataSourceConfiguration(final DataSourceConfiguration config) { + if (config.getValidate()) { + try { + Connection connection = config.buildDatasource().getConnection(); + connection.close(); + } catch (SQLException e) { + throw new IllegalArgumentException( + "Invalid DataSourceConfiguration. Underlying cause: " + e); + } + } + return withDataSourceProviderFn(new DataSourceProviderFromDataSourceConfiguration(config)); + } + + /** + * Setting function that will provide {@link DataSourceConfiguration} in runtime. + * + * @param dataSourceProviderFn a {@link SerializableFunction}. + */ + public Read withDataSourceProviderFn( + SerializableFunction dataSourceProviderFn) { + return toBuilder().setDataSourceProviderFn(dataSourceProviderFn).build(); + } + + /** + * A query to be executed in Snowflake. + * + * @param query - String with query. + */ + public Read fromQuery(String query) { + return toBuilder().setQuery(query).build(); + } + + /** + * A table name to be read in Snowflake. + * + * @param table - String with the name of the table. + */ + public Read fromTable(String table) { + return toBuilder().setTable(table).build(); + } + + /** + * Name of the cloud bucket (GCS by now) to use as tmp location of CSVs during COPY statement. + * + * @param stagingBucketName - String with the name of the bucket. + */ + public Read withStagingBucketName(String stagingBucketName) { + return toBuilder().setStagingBucketName(stagingBucketName).build(); + } + + /** + * Name of the Storage Integration in Snowflake to be used. See + * https://docs.snowflake.com/en/sql-reference/sql/create-storage-integration.html for + * reference. + * + * @param integrationName - String with the name of the Storage Integration. + */ + public Read withIntegrationName(String integrationName) { + return toBuilder().setIntegrationName(integrationName).build(); + } + + /** + * User-defined function mapping CSV lines into user data. + * + * @param csvMapper - an instance of {@link CsvMapper}. + */ + public Read withCsvMapper(CsvMapper csvMapper) { + return toBuilder().setCsvMapper(csvMapper).build(); + } + + /** + * A Coder to be used by the output PCollection generated by the source. + * + * @param coder - an instance of {@link Coder}. + */ + public Read withCoder(Coder coder) { + return toBuilder().setCoder(coder).build(); + } + + @Override + public PCollection expand(PBegin input) { + // Either table or query is required. If query is present, it's being used, table is used + // otherwise + checkArgument( + getQuery() != null || getTable() != null, "fromTable() or fromQuery() is required"); + checkArgument( + !(getQuery() != null && getTable() != null), + "fromTable() and fromQuery() are not allowed together"); + checkArgument(getCsvMapper() != null, "withCsvMapper() is required"); + checkArgument(getCoder() != null, "withCoder() is required"); + checkArgument(getIntegrationName() != null, "withIntegrationName() is required"); + checkArgument(getStagingBucketName() != null, "withStagingBucketName() is required"); + checkArgument( + (getDataSourceProviderFn() != null), + "withDataSourceConfiguration() or withDataSourceProviderFn() is required"); + + String tmpDirName = makeTmpDirName(); + String stagingBucketDir = String.format("%s/%s", getStagingBucketName(), tmpDirName); + + PCollection emptyCollection = input.apply(Create.of((Void) null)); + + PCollection output = + emptyCollection + .apply( + ParDo.of( + new CopyIntoStageFn( + getDataSourceProviderFn(), + getQuery(), + getTable(), + getIntegrationName(), + stagingBucketDir, + getSnowflakeService()))) + .apply(Reshuffle.viaRandomKey()) + .apply(FileIO.matchAll()) + .apply(FileIO.readMatches()) + .apply(readFiles()) + .apply(ParDo.of(new MapCsvToStringArrayFn())) + .apply(ParDo.of(new MapStringArrayToUserDataFn<>(getCsvMapper()))); + + output.setCoder(getCoder()); + + emptyCollection + .apply(Wait.on(output)) + .apply(ParDo.of(new CleanTmpFilesFromGcsFn(stagingBucketDir))); + + return output; + } + + private String makeTmpDirName() { + return String.format( + "sf_copy_csv_%s_%s", + new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()), + UUID.randomUUID().toString().subSequence(0, 8) // first 8 chars of UUID should be enough + ); + } + + private static class CopyIntoStageFn extends DoFn { + private final SerializableFunction dataSourceProviderFn; + private final String query; + private final String table; + private final String integrationName; + private final String stagingBucketDir; + private final SnowflakeService snowflakeService; + + private CopyIntoStageFn( + SerializableFunction dataSourceProviderFn, + String query, + String table, + String integrationName, + String stagingBucketDir, + SnowflakeService snowflakeService) { + this.dataSourceProviderFn = dataSourceProviderFn; + this.query = query; + this.table = table; + this.integrationName = integrationName; + this.stagingBucketDir = + String.format( + "%s/run_%s/", stagingBucketDir, UUID.randomUUID().toString().subSequence(0, 8)); + this.snowflakeService = snowflakeService; + } + + @ProcessElement + public void processElement(ProcessContext context) throws Exception { + String output = + snowflakeService.copyIntoStage( + dataSourceProviderFn, query, table, integrationName, stagingBucketDir); + + context.output(output); + } + } + + public static class MapCsvToStringArrayFn extends DoFn { + @ProcessElement + public void processElement(ProcessContext c) throws IOException { + String csvLine = c.element(); + CSVParser parser = new CSVParserBuilder().withQuoteChar(CSV_QUOTE_CHAR.charAt(0)).build(); + String[] parts = parser.parseLine(csvLine); + c.output(parts); + } + } + + private static class MapStringArrayToUserDataFn extends DoFn { + private final CsvMapper csvMapper; + + public MapStringArrayToUserDataFn(CsvMapper csvMapper) { + this.csvMapper = csvMapper; + } + + @ProcessElement + public void processElement(ProcessContext context) throws Exception { + context.output(csvMapper.mapRow(context.element())); + } + } + + public static class CleanTmpFilesFromGcsFn extends DoFn { + private final String stagingBucketDir; + + public CleanTmpFilesFromGcsFn(String stagingBucketDir) { + this.stagingBucketDir = stagingBucketDir; + } + + @ProcessElement + public void processElement(ProcessContext c) throws IOException { + String combinedPath = stagingBucketDir + "/**"; + List paths = + FileSystems.match(combinedPath).metadata().stream() + .map(metadata -> metadata.resourceId()) + .collect(Collectors.toList()); + + FileSystems.delete(paths, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES); + } + } + + @Override + public void populateDisplayData(DisplayData.Builder builder) { + super.populateDisplayData(builder); + if (getQuery() != null) { + builder.add(DisplayData.item("query", getQuery())); + } + if (getTable() != null) { + builder.add(DisplayData.item("table", getTable())); + } + builder.add(DisplayData.item("integrationName", getIntegrationName())); + builder.add(DisplayData.item("stagingBucketName", getStagingBucketName())); + builder.add(DisplayData.item("csvMapper", getCsvMapper().getClass().getName())); + builder.add(DisplayData.item("coder", getCoder().getClass().getName())); + if (getDataSourceProviderFn() instanceof HasDisplayData) { + ((HasDisplayData) getDataSourceProviderFn()).populateDisplayData(builder); + } + } + } + + /** + * A POJO describing a {@link DataSource}, providing all properties allowing to create a {@link + * DataSource}. + */ + @AutoValue + public abstract static class DataSourceConfiguration implements Serializable { + @Nullable + public abstract String getUrl(); + + @Nullable + public abstract String getUsername(); + + @Nullable + public abstract String getPassword(); + + @Nullable + public abstract PrivateKey getPrivateKey(); + + @Nullable + public abstract String getOauthToken(); + + @Nullable + public abstract String getDatabase(); + + @Nullable + public abstract String getWarehouse(); + + @Nullable + public abstract String getSchema(); + + @Nullable + public abstract String getServerName(); + + @Nullable + public abstract Integer getPortNumber(); + + @Nullable + public abstract String getRole(); + + @Nullable + public abstract Integer getLoginTimeout(); + + @Nullable + public abstract Boolean getSsl(); + + @Nullable + public abstract Boolean getValidate(); + + @Nullable + public abstract DataSource getDataSource(); + + abstract Builder builder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setUrl(String url); + + abstract Builder setUsername(String username); + + abstract Builder setPassword(String password); + + abstract Builder setPrivateKey(PrivateKey privateKey); + + abstract Builder setOauthToken(String oauthToken); + + abstract Builder setDatabase(String database); + + abstract Builder setWarehouse(String warehouse); + + abstract Builder setSchema(String schema); + + abstract Builder setServerName(String serverName); + + abstract Builder setPortNumber(Integer portNumber); + + abstract Builder setRole(String role); + + abstract Builder setLoginTimeout(Integer loginTimeout); + + abstract Builder setSsl(Boolean ssl); + + abstract Builder setValidate(Boolean validate); + + abstract Builder setDataSource(DataSource dataSource); + + abstract DataSourceConfiguration build(); + } + + /** + * Creates {@link DataSourceConfiguration} from existing instance of {@link DataSource}. + * + * @param dataSource - an instance of {@link DataSource}. + */ + public static DataSourceConfiguration create(DataSource dataSource) { + checkArgument(dataSource instanceof Serializable, "dataSource must be Serializable"); + return new AutoValue_SnowflakeIO_DataSourceConfiguration.Builder() + .setValidate(true) + .setDataSource(dataSource) + .build(); + } + + /** + * Creates {@link DataSourceConfiguration} from instance of {@link SnowflakeCredentials}. + * + * @param credentials - an instance of {@link SnowflakeCredentials}. + */ + public static DataSourceConfiguration create(SnowflakeCredentials credentials) { + if (credentials instanceof UsernamePasswordSnowflakeCredentials) { + return new AutoValue_SnowflakeIO_DataSourceConfiguration.Builder() + .setValidate(true) + .setUsername(((UsernamePasswordSnowflakeCredentials) credentials).getUsername()) + .setPassword(((UsernamePasswordSnowflakeCredentials) credentials).getPassword()) + .build(); + } else if (credentials instanceof OAuthTokenSnowflakeCredentials) { + return new AutoValue_SnowflakeIO_DataSourceConfiguration.Builder() + .setValidate(true) + .setOauthToken(((OAuthTokenSnowflakeCredentials) credentials).getToken()) + .build(); + } else if (credentials instanceof KeyPairSnowflakeCredentials) { + return new AutoValue_SnowflakeIO_DataSourceConfiguration.Builder() + .setValidate(true) + .setUsername(((KeyPairSnowflakeCredentials) credentials).getUsername()) + .setPrivateKey(((KeyPairSnowflakeCredentials) credentials).getPrivateKey()) + .build(); + } + throw new IllegalArgumentException( + "Can't create DataSourceConfiguration from given credentials"); + } + + /** + * Sets URL of Snowflake server in following format: + * jdbc:snowflake://.snowflakecomputing.com + * + *

Either withUrl or withServerName is required. + * + * @param url - String with URL of the Snowflake server. + */ + public DataSourceConfiguration withUrl(String url) { + checkArgument( + url.startsWith("jdbc:snowflake://"), + "url must have format: jdbc:snowflake://.snowflakecomputing.com"); + checkArgument( + url.endsWith("snowflakecomputing.com"), + "url must have format: jdbc:snowflake://.snowflakecomputing.com"); + return builder().setUrl(url).build(); + } + + /** + * Sets database to use. + * + * @param database - String with database name. + */ + public DataSourceConfiguration withDatabase(String database) { + return builder().setDatabase(database).build(); + } + + /** + * Sets Snowflake Warehouse to use. + * + * @param warehouse - String with warehouse name. + */ + public DataSourceConfiguration withWarehouse(String warehouse) { + return builder().setWarehouse(warehouse).build(); + } + + /** + * Sets schema to use when connecting to Snowflake. + * + * @param schema - String with schema name. + */ + public DataSourceConfiguration withSchema(String schema) { + return builder().setSchema(schema).build(); + } + + /** + * Sets the name of the Snowflake server. Following format is required: + * .snowflakecomputing.com + * + *

Either withServerName or withUrl is required. + * + * @param serverName - String with server name. + */ + public DataSourceConfiguration withServerName(String serverName) { + checkArgument( + serverName.endsWith("snowflakecomputing.com"), + "serverName must be in format .snowflakecomputing.com"); + return builder().setServerName(serverName).build(); + } + + /** + * Sets port number to use to connect to Snowflake. + * + * @param portNumber - Integer with port number. + */ + public DataSourceConfiguration withPortNumber(Integer portNumber) { + return builder().setPortNumber(portNumber).build(); + } + + /** + * Sets user's role to be used when running queries on Snowflake. + * + * @param role - String with role name. + */ + public DataSourceConfiguration withRole(String role) { + return builder().setRole(role).build(); + } + + /** + * Sets loginTimeout that will be used in {@link SnowflakeBasicDataSource:setLoginTimeout}. + * + * @param loginTimeout - Integer with timeout value. + */ + public DataSourceConfiguration withLoginTimeout(Integer loginTimeout) { + return builder().setLoginTimeout(loginTimeout).build(); + } + + /** + * Disables validation of connection parameters prior to pipeline submission. + * + * @return + */ + public DataSourceConfiguration withoutValidation() { + return builder().setValidate(false).build(); + } + + void populateDisplayData(DisplayData.Builder builder) { + if (getDataSource() != null) { + builder.addIfNotNull(DisplayData.item("dataSource", getDataSource().getClass().getName())); + } else { + builder.addIfNotNull(DisplayData.item("jdbcUrl", getUrl())); + builder.addIfNotNull(DisplayData.item("username", getUsername())); + } + } + + /** Builds {@link SnowflakeBasicDataSource} based on the current configuration. */ + public DataSource buildDatasource() { + if (getDataSource() == null) { + SnowflakeBasicDataSource basicDataSource = new SnowflakeBasicDataSource(); + + if (getUrl() != null) { + basicDataSource.setUrl(getUrl()); + } + if (getUsername() != null) { + basicDataSource.setUser(getUsername()); + } + if (getPassword() != null) { + basicDataSource.setPassword(getPassword()); + } + if (getPrivateKey() != null) { + basicDataSource.setPrivateKey(getPrivateKey()); + } + if (getDatabase() != null) { + basicDataSource.setDatabaseName(getDatabase()); + } + if (getWarehouse() != null) { + basicDataSource.setWarehouse(getWarehouse()); + } + if (getSchema() != null) { + basicDataSource.setSchema(getSchema()); + } + if (getServerName() != null) { + basicDataSource.setServerName(getServerName()); + } + if (getPortNumber() != null) { + basicDataSource.setPortNumber(getPortNumber()); + } + if (getRole() != null) { + basicDataSource.setRole(getRole()); + } + if (getLoginTimeout() != null) { + try { + basicDataSource.setLoginTimeout(getLoginTimeout()); + } catch (SQLException e) { + throw new RuntimeException("Failed to setLoginTimeout"); + } + } + if (getOauthToken() != null) { + basicDataSource.setOauthToken(getOauthToken()); + } + return basicDataSource; + } + return getDataSource(); + } + } + + public static class DataSourceProviderFromDataSourceConfiguration + implements SerializableFunction, HasDisplayData { + private static final ConcurrentHashMap instances = + new ConcurrentHashMap<>(); + private final DataSourceConfiguration config; + + private DataSourceProviderFromDataSourceConfiguration(DataSourceConfiguration config) { + this.config = config; + } + + public static SerializableFunction of(DataSourceConfiguration config) { + return new DataSourceProviderFromDataSourceConfiguration(config); + } + + @Override + public DataSource apply(Void input) { + return instances.computeIfAbsent(config, (config) -> config.buildDatasource()); + } + + @Override + public void populateDisplayData(DisplayData.Builder builder) { + config.populateDisplayData(builder); + } + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakePipelineOptions.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakePipelineOptions.java new file mode 100644 index 000000000000..783230edbb8d --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakePipelineOptions.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake; + +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.Validation; + +public interface SnowflakePipelineOptions extends PipelineOptions { + String BASIC_CONNECTION_INFO_VALIDATION_GROUP = "BASIC_CONNECTION_INFO_GROUP"; + String AUTH_VALIDATION_GROUP = "AUTH_VALIDATION_GROUP"; + + @Description( + "Snowflake's JDBC-like url including account name and region without any parameters.") + @Validation.Required(groups = BASIC_CONNECTION_INFO_VALIDATION_GROUP) + String getUrl(); + + void setUrl(String url); + + @Description("Server Name - full server name with account, zone and domain.") + @Validation.Required(groups = BASIC_CONNECTION_INFO_VALIDATION_GROUP) + String getServerName(); + + void setServerName(String serverName); + + @Description("Username. Required for username/password and Private Key authentication.") + @Validation.Required(groups = AUTH_VALIDATION_GROUP) + String getUsername(); + + void setUsername(String username); + + @Description("OAuth token. Required for OAuth authentication only.") + @Validation.Required(groups = AUTH_VALIDATION_GROUP) + String getOauthToken(); + + void setOauthToken(String oauthToken); + + @Description("Password. Required for username/password authentication only.") + @Default.String("") + String getPassword(); + + void setPassword(String password); + + @Description("Path to Private Key file. Required for Private Key authentication only.") + @Default.String("") + String getPrivateKeyPath(); + + void setPrivateKeyPath(String privateKeyPath); + + @Description("Private Key's passphrase. Required for Private Key authentication only.") + @Default.String("") + String getPrivateKeyPassphrase(); + + void setPrivateKeyPassphrase(String keyPassphrase); + + @Description("Warehouse to use. Optional.") + @Default.String("") + String getWarehouse(); + + void setWarehouse(String warehouse); + + @Description("Database name to connect to. Optional.") + @Default.String("") + String getDatabase(); + + void setDatabase(String database); + + @Description("Schema to use. Optional.") + @Default.String("") + String getSchema(); + + void setSchema(String schema); + + @Description("Role to use. Optional.") + @Default.String("") + String getRole(); + + void setRole(String role); + + @Description("Authenticator to use. Optional.") + @Default.String("") + String getAuthenticator(); + + void setAuthenticator(String authenticator); + + @Description("Port number. Optional.") + @Default.String("") + String getPortNumber(); + + void setPortNumber(String portNumber); + + @Description("Login timeout. Optional.") + @Default.String("") + String getLoginTimeout(); + + void setLoginTimeout(String loginTimeout); + + @Description("External location name to connect to.") + String getExternalLocation(); + + void setExternalLocation(String externalLocation); + + @Description("Temporary GCS bucket name") + String getStagingBucketName(); + + void setStagingBucketName(String stagingBucketName); + + @Description("Storage integration - required in case the external stage is not specified.") + String getStorageIntegration(); + + void setStorageIntegration(String integration); + + @Description("Stage name. Optional.") + String getStage(); + + void setStage(String stage); +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeService.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeService.java new file mode 100644 index 000000000000..6375e79e74ce --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeService.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake; + +import java.io.Serializable; +import java.sql.SQLException; +import javax.sql.DataSource; +import org.apache.beam.sdk.transforms.SerializableFunction; + +/** Interface which defines common methods for interacting with SnowFlake. */ +public interface SnowflakeService extends Serializable { + String CSV_QUOTE_CHAR_FOR_COPY = "''"; + + String copyIntoStage( + SerializableFunction dataSourceProviderFn, + String query, + String table, + String integrationName, + String stagingBucketDir) + throws SQLException; +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeServiceImpl.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeServiceImpl.java new file mode 100644 index 000000000000..5aaad06b7ed2 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeServiceImpl.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.function.Consumer; +import javax.sql.DataSource; +import org.apache.beam.sdk.transforms.SerializableFunction; + +/** + * Implemenation of {@link org.apache.beam.sdk.io.snowflake.SnowflakeService} used in production. + */ +public class SnowflakeServiceImpl implements SnowflakeService { + private static final String SNOWFLAKE_GCS_PREFIX = "gcs://"; + + @Override + public String copyIntoStage( + SerializableFunction dataSourceProviderFn, + String query, + String table, + String integrationName, + String stagingBucketDir) + throws SQLException { + + String from; + if (query != null) { + // Query must be surrounded with brackets + from = String.format("(%s)", query); + } else { + from = table; + } + + String copyQuery = + String.format( + "COPY INTO '%s' FROM %s STORAGE_INTEGRATION=%s FILE_FORMAT=(TYPE=CSV COMPRESSION=GZIP FIELD_OPTIONALLY_ENCLOSED_BY='%s');", + getProperBucketDir(stagingBucketDir), from, integrationName, CSV_QUOTE_CHAR_FOR_COPY); + + runStatement(copyQuery, getConnection(dataSourceProviderFn), null); + + return stagingBucketDir.concat("*"); + } + + private static void runStatement(String query, Connection connection, Consumer resultSetMethod) + throws SQLException { + PreparedStatement statement = connection.prepareStatement(query); + try { + if (resultSetMethod != null) { + ResultSet resultSet = statement.executeQuery(); + resultSetMethod.accept(resultSet); + } else { + statement.execute(); + } + } finally { + statement.close(); + connection.close(); + } + } + + private Connection getConnection(SerializableFunction dataSourceProviderFn) + throws SQLException { + DataSource dataSource = dataSourceProviderFn.apply(null); + return dataSource.getConnection(); + } + + // Snowflake is expecting "gcs://" prefix for GCS and Beam "gs://" + private String getProperBucketDir(String bucketDir) { + if (bucketDir.contains(CloudProvider.GCS.getPrefix())) { + return bucketDir.replace(CloudProvider.GCS.getPrefix(), SNOWFLAKE_GCS_PREFIX); + } + return bucketDir; + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/KeyPairSnowflakeCredentials.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/KeyPairSnowflakeCredentials.java new file mode 100644 index 000000000000..286ec628fc8b --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/KeyPairSnowflakeCredentials.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.credentials; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.InvalidKeyException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Base64; +import javax.crypto.EncryptedPrivateKeyInfo; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; + +/** POJO for handling Key-Pair authentication against Snowflake. */ +public class KeyPairSnowflakeCredentials implements SnowflakeCredentials { + private String username; + private PrivateKey privateKey; + + public KeyPairSnowflakeCredentials( + String username, String privateKeyPath, String privateKeyPassword) { + this.username = username; + this.privateKey = getPrivateKey(privateKeyPath, privateKeyPassword); + } + + public KeyPairSnowflakeCredentials(String username, PrivateKey privateKey) { + this.username = username; + this.privateKey = privateKey; + } + + private PrivateKey getPrivateKey(String privateKeyPath, String privateKeyPassphrase) { + try { + byte[] keyBytes = Files.readAllBytes(Paths.get(privateKeyPath)); + + String encrypted = new String(keyBytes, Charset.defaultCharset()); + encrypted = encrypted.replace("-----BEGIN ENCRYPTED PRIVATE KEY-----", ""); + encrypted = encrypted.replace("-----END ENCRYPTED PRIVATE KEY-----", ""); + EncryptedPrivateKeyInfo pkInfo = + new EncryptedPrivateKeyInfo(Base64.getMimeDecoder().decode(encrypted)); + PBEKeySpec keySpec = new PBEKeySpec(privateKeyPassphrase.toCharArray()); + SecretKeyFactory pbeKeyFactory = SecretKeyFactory.getInstance(pkInfo.getAlgName()); + PKCS8EncodedKeySpec encodedKeySpec = pkInfo.getKeySpec(pbeKeyFactory.generateSecret(keySpec)); + + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return keyFactory.generatePrivate(encodedKeySpec); + } catch (IOException + | NoSuchAlgorithmException + | InvalidKeySpecException + | InvalidKeyException ex) { + throw new RuntimeException("Can't create PrivateKey from options"); + } + } + + public String getUsername() { + return username; + } + + public PrivateKey getPrivateKey() { + return privateKey; + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/OAuthTokenSnowflakeCredentials.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/OAuthTokenSnowflakeCredentials.java new file mode 100644 index 000000000000..be102a8b0f41 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/OAuthTokenSnowflakeCredentials.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.credentials; + +/** POJO for handling OAuth authentication against Snowflake, using pre-obtained OAuth token. */ +public class OAuthTokenSnowflakeCredentials implements SnowflakeCredentials { + private String token; + + public OAuthTokenSnowflakeCredentials(String token) { + this.token = token; + } + + public String getToken() { + return token; + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentials.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentials.java new file mode 100644 index 000000000000..e3abf91f7d12 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentials.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.credentials; + +/** + * Interface for holding credentials. Allows creating {@link + * org.apache.beam.sdk.io.snowflake.SnowflakeIO.DataSourceConfiguration}. + */ +public interface SnowflakeCredentials {} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentialsFactory.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentialsFactory.java new file mode 100644 index 000000000000..3876c2f10d71 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/SnowflakeCredentialsFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.credentials; + +import org.apache.beam.sdk.io.snowflake.SnowflakePipelineOptions; + +/** + * Factory class for creating implementations of {@link SnowflakeCredentials} from {@link + * SnowflakePipelineOptions}. + */ +public class SnowflakeCredentialsFactory { + public static SnowflakeCredentials of(SnowflakePipelineOptions options) { + if (oauthOptionsAvailable(options)) { + return new OAuthTokenSnowflakeCredentials(options.getOauthToken()); + } else if (usernamePasswordOptionsAvailable(options)) { + return new UsernamePasswordSnowflakeCredentials(options.getUsername(), options.getPassword()); + } else if (keyPairOptionsAvailable(options)) { + return new KeyPairSnowflakeCredentials( + options.getUsername(), options.getPrivateKeyPath(), options.getPrivateKeyPassphrase()); + } + throw new RuntimeException("Can't get credentials from Options"); + } + + private static boolean oauthOptionsAvailable(SnowflakePipelineOptions options) { + return options.getOauthToken() != null && !options.getOauthToken().isEmpty(); + } + + private static boolean usernamePasswordOptionsAvailable(SnowflakePipelineOptions options) { + return options.getUsername() != null + && !options.getUsername().isEmpty() + && !options.getPassword().isEmpty(); + } + + private static boolean keyPairOptionsAvailable(SnowflakePipelineOptions options) { + return options.getUsername() != null + && !options.getUsername().isEmpty() + && !options.getPrivateKeyPath().isEmpty() + && !options.getPrivateKeyPassphrase().isEmpty(); + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/UsernamePasswordSnowflakeCredentials.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/UsernamePasswordSnowflakeCredentials.java new file mode 100644 index 000000000000..1d8bdce0cc2e --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/UsernamePasswordSnowflakeCredentials.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.credentials; + +/** POJO for handling Username & Password authentication against Snowflake. */ +public class UsernamePasswordSnowflakeCredentials implements SnowflakeCredentials { + private String username; + private String password; + + public UsernamePasswordSnowflakeCredentials(String username, String password) { + this.username = username; + this.password = password; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } +} diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/package-info.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/package-info.java new file mode 100644 index 000000000000..f76d2419b8ce --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/credentials/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Credentials for SnowflakeIO. */ +package org.apache.beam.sdk.io.snowflake.credentials; diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/package-info.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/package-info.java new file mode 100644 index 000000000000..9dbcf05a4074 --- /dev/null +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Snowflake IO transforms. */ +package org.apache.beam.sdk.io.snowflake; diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeBasicDataSource.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeBasicDataSource.java new file mode 100644 index 000000000000..5fc694fb9d0c --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeBasicDataSource.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test; + +import java.io.Serializable; +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.NClob; +import java.sql.PreparedStatement; +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Savepoint; +import java.sql.Statement; +import java.sql.Struct; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Executor; +import net.snowflake.client.jdbc.SnowflakeBasicDataSource; + +/** + * Fake implementation of {@link net.snowflake.client.jdbc.SnowflakeBasicDataSource} used in tests. + */ +public class FakeSnowflakeBasicDataSource extends SnowflakeBasicDataSource implements Serializable { + @Override + public FakeConnection getConnection() throws SQLException { + return new FakeConnection(); + } + + private class FakeConnection implements Connection { + + @Override + public Statement createStatement() throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql) throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + return null; + } + + @Override + public String nativeSQL(String sql) throws SQLException { + return null; + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException {} + + @Override + public boolean getAutoCommit() throws SQLException { + return false; + } + + @Override + public void commit() throws SQLException {} + + @Override + public void rollback() throws SQLException {} + + @Override + public void close() throws SQLException {} + + @Override + public boolean isClosed() throws SQLException { + return false; + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return null; + } + + @Override + public void setReadOnly(boolean readOnly) throws SQLException {} + + @Override + public boolean isReadOnly() throws SQLException { + return false; + } + + @Override + public void setCatalog(String catalog) throws SQLException {} + + @Override + public String getCatalog() throws SQLException { + return null; + } + + @Override + public void setTransactionIsolation(int level) throws SQLException {} + + @Override + public int getTransactionIsolation() throws SQLException { + return 0; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return null; + } + + @Override + public void clearWarnings() throws SQLException {} + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) + throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement( + String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + return null; + } + + @Override + public Map> getTypeMap() throws SQLException { + return null; + } + + @Override + public void setTypeMap(Map> map) throws SQLException {} + + @Override + public void setHoldability(int holdability) throws SQLException {} + + @Override + public int getHoldability() throws SQLException { + return 0; + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return null; + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + return null; + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException {} + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException {} + + @Override + public Statement createStatement( + int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement( + String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall( + String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) + throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) + throws SQLException { + return null; + } + + @Override + public Clob createClob() throws SQLException { + return null; + } + + @Override + public Blob createBlob() throws SQLException { + return null; + } + + @Override + public NClob createNClob() throws SQLException { + return null; + } + + @Override + public SQLXML createSQLXML() throws SQLException { + return null; + } + + @Override + public boolean isValid(int timeout) throws SQLException { + return false; + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException {} + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException {} + + @Override + public String getClientInfo(String name) throws SQLException { + return null; + } + + @Override + public Properties getClientInfo() throws SQLException { + return null; + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + return null; + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + return null; + } + + @Override + public void setSchema(String schema) throws SQLException {} + + @Override + public String getSchema() throws SQLException { + return null; + } + + @Override + public void abort(Executor executor) throws SQLException {} + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {} + + @Override + public int getNetworkTimeout() throws SQLException { + return 0; + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeDatabase.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeDatabase.java new file mode 100644 index 000000000000..5bf8b21afa18 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeDatabase.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test; + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import net.snowflake.client.jdbc.SnowflakeSQLException; + +/** Fake implementation of SnowFlake warehouse used in test code. */ +public class FakeSnowflakeDatabase implements Serializable { + private static Map> tables = new HashMap<>(); + + private FakeSnowflakeDatabase() { + tables = new HashMap<>(); + } + + public static void createTable(String table) { + FakeSnowflakeDatabase.tables.put(table, Collections.emptyList()); + } + + public static List getElements(String table) throws SnowflakeSQLException { + if (!isTableExist(table)) { + throw new SnowflakeSQLException( + null, "SQL compilation error: Table does not exist", table, 0); + } + + return FakeSnowflakeDatabase.tables.get(table); + } + + public static List runQuery(String query) throws SnowflakeSQLException { + if (query.startsWith("SELECT * FROM ")) { + String tableName = query.replace("SELECT * FROM ", ""); + return getElements(tableName); + } + throw new SnowflakeSQLException(null, "SQL compilation error: Invalid query", query, 0); + } + + public static List getElementsAsLong(String table) throws SnowflakeSQLException { + List elements = getElements(table); + return elements.stream().map(Long::parseLong).collect(Collectors.toList()); + } + + public static boolean isTableExist(String table) { + return FakeSnowflakeDatabase.tables.containsKey(table); + } + + public static boolean isTableEmpty(String table) { + return FakeSnowflakeDatabase.tables.get(table).isEmpty(); + } + + public static void createTableWithElements(String table, List rows) { + FakeSnowflakeDatabase.tables.put(table, rows); + } + + public static void clean() { + FakeSnowflakeDatabase.tables = new HashMap<>(); + } + + public static void truncateTable(String table) { + FakeSnowflakeDatabase.createTable(table); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeServiceImpl.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeServiceImpl.java new file mode 100644 index 000000000000..4a62dcd3181d --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeServiceImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.util.List; +import javax.sql.DataSource; +import org.apache.beam.sdk.io.snowflake.SnowflakeService; +import org.apache.beam.sdk.transforms.SerializableFunction; + +/** + * Fake implementation of {@link org.apache.beam.sdk.io.snowflake.SnowflakeService} used in tests. + */ +public class FakeSnowflakeServiceImpl implements SnowflakeService { + + @Override + public String copyIntoStage( + SerializableFunction dataSourceProviderFn, + String query, + String table, + String integrationName, + String stagingBucketName) + throws SQLException { + + if (table != null) { + writeToFile(stagingBucketName, FakeSnowflakeDatabase.getElements(table)); + } + if (query != null) { + writeToFile(stagingBucketName, FakeSnowflakeDatabase.runQuery(query)); + } + + return String.format("./%s/*", stagingBucketName); + } + + private void writeToFile(String stagingBucketNameTmp, List rows) { + Path filePath = Paths.get(String.format("./%s/table.csv.gz", stagingBucketNameTmp)); + try { + Files.createDirectories(filePath.getParent()); + Files.createFile(filePath); + Files.write(filePath, rows); + } catch (IOException e) { + throw new RuntimeException("Failed to create files", e); + } + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/TestUtils.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/TestUtils.java new file mode 100644 index 000000000000..aab8d7de9660 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/TestUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test; + +import java.io.File; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestUtils { + + private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class); + + private static final String PRIVATE_KEY_FILE_NAME = "test_rsa_key.p8"; + private static final String PRIVATE_KEY_PASSPHRASE = "snowflake"; + + public static String getPrivateKeyPath(Class klass) { + ClassLoader classLoader = klass.getClassLoader(); + File file = new File(classLoader.getResource(PRIVATE_KEY_FILE_NAME).getFile()); + return file.getAbsolutePath(); + } + + public static String getPrivateKeyPassphrase() { + return PRIVATE_KEY_PASSPHRASE; + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/package-info.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/package-info.java new file mode 100644 index 000000000000..2e2cf2f7645c --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Snowflake IO tests. */ +package org.apache.beam.sdk.io.snowflake.test; diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/BatchTestPipelineOptions.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/BatchTestPipelineOptions.java new file mode 100644 index 000000000000..3504c455851b --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/BatchTestPipelineOptions.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit; + +import org.apache.beam.sdk.io.snowflake.SnowflakePipelineOptions; +import org.apache.beam.sdk.options.Description; + +public interface BatchTestPipelineOptions extends SnowflakePipelineOptions { + @Description("Table name to connect to.") + String getTable(); + + void setTable(String table); +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/DataSourceConfigurationTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/DataSourceConfigurationTest.java new file mode 100644 index 000000000000..98f22132c77f --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/DataSourceConfigurationTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import javax.sql.DataSource; +import net.snowflake.client.jdbc.SnowflakeBasicDataSource; +import org.apache.beam.sdk.io.snowflake.SnowflakeIO; +import org.apache.beam.sdk.io.snowflake.credentials.OAuthTokenSnowflakeCredentials; +import org.junit.Before; +import org.junit.Test; + +/** Unit tests for {@link org.apache.beam.sdk.io.snowflake.SnowflakeIO.DataSourceConfiguration}. */ +public class DataSourceConfigurationTest { + + private SnowflakeIO.DataSourceConfiguration configuration; + + @Before + public void setUp() { + configuration = + SnowflakeIO.DataSourceConfiguration.create( + new OAuthTokenSnowflakeCredentials("some-token")); + } + + @Test + public void testSettingUrlWithBadPrefix() { + assertThrows( + IllegalArgumentException.class, + () -> configuration.withUrl("account.snowflakecomputing.com")); + } + + @Test + public void testSettingUrlWithBadSuffix() { + assertThrows( + IllegalArgumentException.class, () -> configuration.withUrl("jdbc:snowflake://account")); + } + + @Test + public void testSettingStringUrl() { + String url = "jdbc:snowflake://account.snowflakecomputing.com"; + configuration = configuration.withUrl(url); + assertEquals(url, configuration.getUrl()); + } + + @Test + public void testSettingServerNameWithBadSuffix() { + assertThrows( + IllegalArgumentException.class, () -> configuration.withServerName("not.properly.ended")); + } + + @Test + public void testSettingStringServerName() { + String serverName = "account.snowflakecomputing.com"; + configuration = configuration.withServerName(serverName); + assertEquals(serverName, configuration.getServerName()); + } + + @Test + public void testSettingStringDatabase() { + String database = "dbname"; + configuration = configuration.withDatabase(database); + assertEquals(database, configuration.getDatabase()); + } + + @Test + public void testSettingStringWarehouse() { + String warehouse = "warehouse"; + configuration = configuration.withWarehouse(warehouse); + assertEquals(warehouse, configuration.getWarehouse()); + } + + @Test + public void testSettingStringSchema() { + String schema = "schema"; + configuration = configuration.withSchema(schema); + assertEquals(schema, configuration.getSchema()); + } + + @Test + public void testSettingStringRole() { + String role = "role"; + configuration = configuration.withRole(role); + assertEquals(role, configuration.getRole()); + } + + @Test + public void testSettingStringPortNumber() { + Integer portNumber = 1234; + configuration = configuration.withPortNumber(portNumber); + assertEquals(portNumber, configuration.getPortNumber()); + } + + @Test + public void testSettingStringLoginTimeout() { + Integer loginTimeout = 999; + configuration = configuration.withLoginTimeout(loginTimeout); + assertEquals(loginTimeout, configuration.getLoginTimeout()); + } + + @Test + public void testSettingValidate() { + configuration = configuration.withoutValidation(); + assertEquals(false, configuration.getValidate()); + } + + @Test + public void testDataSourceCreatedFromUrl() { + String url = "jdbc:snowflake://account.snowflakecomputing.com"; + configuration = configuration.withUrl(url); + + DataSource dataSource = configuration.buildDatasource(); + + assertEquals(SnowflakeBasicDataSource.class, dataSource.getClass()); + assertEquals(url, ((SnowflakeBasicDataSource) dataSource).getUrl()); + } + + @Test + public void testDataSourceCreatedFromServerName() { + String serverName = "account.snowflakecomputing.com"; + configuration = configuration.withServerName(serverName); + + DataSource dataSource = configuration.buildDatasource(); + + String expectedUrl = "jdbc:snowflake://account.snowflakecomputing.com"; + assertEquals(SnowflakeBasicDataSource.class, dataSource.getClass()); + assertEquals(expectedUrl, ((SnowflakeBasicDataSource) dataSource).getUrl()); + } + + @Test + public void testDataSourceCreatedFromServerNameAndPort() { + String serverName = "account.snowflakecomputing.com"; + int portNumber = 1234; + + configuration = configuration.withServerName(serverName); + configuration = configuration.withPortNumber(portNumber); + + DataSource dataSource = configuration.buildDatasource(); + assertEquals(SnowflakeBasicDataSource.class, dataSource.getClass()); + String expectedUrl = "jdbc:snowflake://account.snowflakecomputing.com:1234"; + assertEquals(expectedUrl, ((SnowflakeBasicDataSource) dataSource).getUrl()); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/KeyPairSnowflakeCredentialsTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/KeyPairSnowflakeCredentialsTest.java new file mode 100644 index 000000000000..b231a2dde665 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/KeyPairSnowflakeCredentialsTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit.credentials; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.beam.sdk.io.snowflake.credentials.KeyPairSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.test.TestUtils; +import org.junit.Test; + +public class KeyPairSnowflakeCredentialsTest { + @Test + public void testFilePathConstructor() { + KeyPairSnowflakeCredentials credentials = + new KeyPairSnowflakeCredentials( + "username", + TestUtils.getPrivateKeyPath(getClass()), + TestUtils.getPrivateKeyPassphrase()); + assertEquals("username", credentials.getUsername()); + assertNotNull(credentials.getPrivateKey()); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/OAuthTokenSnowflakeCredentialsTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/OAuthTokenSnowflakeCredentialsTest.java new file mode 100644 index 000000000000..a1dee76f1601 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/OAuthTokenSnowflakeCredentialsTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit.credentials; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.beam.sdk.io.snowflake.SnowflakeIO; +import org.apache.beam.sdk.io.snowflake.credentials.OAuthTokenSnowflakeCredentials; +import org.junit.Test; + +public class OAuthTokenSnowflakeCredentialsTest { + + @Test + public void testConstructor() { + OAuthTokenSnowflakeCredentials credentials = new OAuthTokenSnowflakeCredentials("token"); + + assertEquals("token", credentials.getToken()); + } + + @Test + public void testBuildingDataSource() { + OAuthTokenSnowflakeCredentials credentials = new OAuthTokenSnowflakeCredentials("token"); + + SnowflakeIO.DataSourceConfiguration configuration = + SnowflakeIO.DataSourceConfiguration.create(credentials); + + assertEquals(credentials.getToken(), configuration.getOauthToken()); + assertTrue(configuration.getValidate()); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/SnowflakeCredentialsFactoryTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/SnowflakeCredentialsFactoryTest.java new file mode 100644 index 000000000000..f9f612d8f002 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/SnowflakeCredentialsFactoryTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit.credentials; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.beam.sdk.io.snowflake.SnowflakePipelineOptions; +import org.apache.beam.sdk.io.snowflake.credentials.KeyPairSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.OAuthTokenSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.SnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.credentials.SnowflakeCredentialsFactory; +import org.apache.beam.sdk.io.snowflake.credentials.UsernamePasswordSnowflakeCredentials; +import org.apache.beam.sdk.io.snowflake.test.TestUtils; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.junit.Test; + +public class SnowflakeCredentialsFactoryTest { + + @Test + public void usernamePasswordTest() { + SnowflakePipelineOptions options = PipelineOptionsFactory.as(SnowflakePipelineOptions.class); + options.setUsername("username"); + options.setPassword("password"); + + SnowflakeCredentials credentials = SnowflakeCredentialsFactory.of(options); + + assertEquals(UsernamePasswordSnowflakeCredentials.class, credentials.getClass()); + } + + @Test + public void oauthTokenTest() { + SnowflakePipelineOptions options = PipelineOptionsFactory.as(SnowflakePipelineOptions.class); + options.setOauthToken("token"); + + SnowflakeCredentials credentials = SnowflakeCredentialsFactory.of(options); + + assertEquals(OAuthTokenSnowflakeCredentials.class, credentials.getClass()); + } + + @Test + public void keyPairTest() { + SnowflakePipelineOptions options = PipelineOptionsFactory.as(SnowflakePipelineOptions.class); + System.out.println(TestUtils.getPrivateKeyPath(getClass())); + options.setUsername("username"); + options.setPrivateKeyPath(TestUtils.getPrivateKeyPath(getClass())); + options.setPrivateKeyPassphrase(TestUtils.getPrivateKeyPassphrase()); + + SnowflakeCredentials credentials = SnowflakeCredentialsFactory.of(options); + + assertEquals(KeyPairSnowflakeCredentials.class, credentials.getClass()); + } + + @Test + public void emptyOptionsTest() { + SnowflakePipelineOptions options = PipelineOptionsFactory.as(SnowflakePipelineOptions.class); + + Exception ex = + assertThrows(RuntimeException.class, () -> SnowflakeCredentialsFactory.of(options)); + assertEquals("Can't get credentials from Options", ex.getMessage()); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/UsernamePasswordSnowflakeCredentialsTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/UsernamePasswordSnowflakeCredentialsTest.java new file mode 100644 index 000000000000..0c7503a079b8 --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/credentials/UsernamePasswordSnowflakeCredentialsTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit.credentials; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.beam.sdk.io.snowflake.SnowflakeIO; +import org.apache.beam.sdk.io.snowflake.credentials.UsernamePasswordSnowflakeCredentials; +import org.junit.Test; + +public class UsernamePasswordSnowflakeCredentialsTest { + + @Test + public void testConstructor() { + UsernamePasswordSnowflakeCredentials credentials = + new UsernamePasswordSnowflakeCredentials("username", "password"); + + assertEquals("username", credentials.getUsername()); + assertEquals("password", credentials.getPassword()); + } + + @Test + public void testBuildingDataSource() { + UsernamePasswordSnowflakeCredentials credentials = + new UsernamePasswordSnowflakeCredentials("username", "password"); + + SnowflakeIO.DataSourceConfiguration configuration = + SnowflakeIO.DataSourceConfiguration.create(credentials); + + assertEquals("username", configuration.getUsername()); + assertEquals("password", configuration.getPassword()); + assertTrue(configuration.getValidate()); + } +} diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java new file mode 100644 index 000000000000..e4eda0d7520b --- /dev/null +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.snowflake.test.unit.read; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.beam.sdk.Pipeline.PipelineExecutionException; +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.io.AvroGeneratedUser; +import org.apache.beam.sdk.io.snowflake.SnowflakeIO; +import org.apache.beam.sdk.io.snowflake.SnowflakeService; +import org.apache.beam.sdk.io.snowflake.test.FakeSnowflakeBasicDataSource; +import org.apache.beam.sdk.io.snowflake.test.FakeSnowflakeDatabase; +import org.apache.beam.sdk.io.snowflake.test.FakeSnowflakeServiceImpl; +import org.apache.beam.sdk.io.snowflake.test.unit.BatchTestPipelineOptions; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SnowflakeIOReadTest implements Serializable { + public static final String FAKE_TABLE = "FAKE_TABLE"; + public static final String FAKE_QUERY = "SELECT * FROM FAKE_TABLE"; + + private static final BatchTestPipelineOptions options = + TestPipeline.testingPipelineOptions().as(BatchTestPipelineOptions.class);; + @Rule public final transient TestPipeline pipeline = TestPipeline.create(); + + @Rule public transient ExpectedException thrown = ExpectedException.none(); + + private static SnowflakeIO.DataSourceConfiguration dataSourceConfiguration; + private static SnowflakeService snowflakeService; + private static String stagingBucketName; + private static String integrationName; + private static List avroTestData; + + @BeforeClass + public static void setup() { + + List testData = Arrays.asList("Paul,51,red", "Jackson,41,green"); + + avroTestData = + ImmutableList.of( + new AvroGeneratedUser("Paul", 51, "red"), + new AvroGeneratedUser("Jackson", 41, "green")); + + FakeSnowflakeDatabase.createTableWithElements(FAKE_TABLE, testData); + + options.setServerName("NULL.snowflakecomputing.com"); + options.setStorageIntegration("STORAGE_INTEGRATION"); + options.setStagingBucketName("BUCKET"); + + stagingBucketName = options.getStagingBucketName(); + integrationName = options.getStorageIntegration(); + + dataSourceConfiguration = + SnowflakeIO.DataSourceConfiguration.create(new FakeSnowflakeBasicDataSource()) + .withServerName(options.getServerName()); + + snowflakeService = new FakeSnowflakeServiceImpl(); + } + + @Test + public void testConfigIsMissingStagingBucketName() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("withStagingBucketName() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable(FAKE_TABLE) + .withIntegrationName(integrationName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testConfigIsMissingIntegrationName() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("withIntegrationName() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testConfigIsMissingCsvMapper() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("withCsvMapper() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testConfigIsMissingCoder() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("withCoder() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper())); + + pipeline.run(); + } + + @Test + public void testConfigIsMissingFromTableOrFromQuery() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("fromTable() or fromQuery() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testConfigIsMissingDataSourceConfiguration() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("withDataSourceConfiguration() or withDataSourceProviderFn() is required"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testConfigContainsFromQueryAndFromTable() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("fromTable() and fromQuery() are not allowed together"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromQuery("") + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testTableDoesntExist() { + thrown.expect(PipelineExecutionException.class); + thrown.expectMessage("SQL compilation error: Table does not exist"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable("NON_EXIST") + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testInvalidQuery() { + thrown.expect(PipelineExecutionException.class); + thrown.expectMessage("SQL compilation error: Invalid query"); + + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromQuery("BAD_QUERY") + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + pipeline.run(); + } + + @Test + public void testReadFromTable() { + PCollection items = + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromTable(FAKE_TABLE) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + PAssert.that(items).containsInAnyOrder(avroTestData); + pipeline.run(); + } + + @Test + public void testReadFromQuery() { + PCollection items = + pipeline.apply( + SnowflakeIO.read(snowflakeService) + .withDataSourceConfiguration(dataSourceConfiguration) + .fromQuery(FAKE_QUERY) + .withStagingBucketName(stagingBucketName) + .withIntegrationName(integrationName) + .withCsvMapper(getCsvMapper()) + .withCoder(AvroCoder.of(AvroGeneratedUser.getClassSchema()))); + + PAssert.that(items).containsInAnyOrder(avroTestData); + pipeline.run(); + } + + static SnowflakeIO.CsvMapper getCsvMapper() { + return (SnowflakeIO.CsvMapper) + parts -> + new GenericRecordBuilder(AvroGeneratedUser.getClassSchema()) + .set("name", String.valueOf(parts[0])) + .set("favorite_number", Integer.valueOf(parts[1])) + .set("favorite_color", String.valueOf(parts[2])) + .build(); + } +} diff --git a/sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8 b/sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8 new file mode 100644 index 000000000000..ee86a8dd39dd --- /dev/null +++ b/sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8 @@ -0,0 +1,29 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIE6TAbBgkqhkiG9w0BBQMwDgQI2sbwjlr6RrcCAggABIIEyPbY/oiy8EH1QY9t +EVlG2GyZK07bYsVP57PfFWVTJi6gN7G8zxF8vQvPN2fm8w1GJTzyz+ZAdxAXEv6/ +Oe0KmvYbe+YeO0+u+aZah8rnhFgAo1OgMDA8hCKLL98qrGau03TRGZKD6Xxce3nZ +DizFbCxcRejnJflWUFWgyro45Qnb9jtOop+rnmqHeV/CMP/RLattYsZ0SrcCnr1O +fIfb7jqNjY94V7xh9O8G1g5YHL3fv8ir3iXLpy7wPjMVHSMGcK092tnAK9/okmiA +EdzNH/DfwUQ2qZ4gWJJp75b5RwVTZ9uNDQZkQJQLAS8zowtjXhKJ+Zy26ID/EI7u +H5R5WhUl2ROI2ssjX8I0wFyOMA5guBjw2xL/Y+/eI3dQ+2g6hxN2TpPoKEAzeM1a +OyrTyyVZ7nJyJ7RC6odRnHE9PLFzpmGv/3YhbSvhAwAnLN2mky+LqSw8hTGVMaIv +QbnkeqxGGd6miIZgPOrbBj1mRErLdgKkJn9UNYPXKoB+0gEyLGYSkY1TdJ6NjRq8 +5oU+MgU1dnyp73JMiafz4AbFLHrxXG3AG6vww9WBgiV9pFmQSMcuVX8p8rvxJ8H4 +nFQlwiZrvl98xcfGICkoiSKP2fEDir1sdbjpcEY1Rxk3baUqBxZCrRKo+Dz4k4yy +VjZX8SkXrYgcQNxmDiv59D34QYtKyq8ZxeHB7tVrj8/G1N8WgU8EApR0+yQL+0TP +8aicPc++9ta62Wv59iqmsQeR+Bdq1/kOTZCMA9QMK7/mJpcn18/5EmbrAUbw/2pf +TdyLqEXUf3N0zrDUAKUuWSOJMHOEhdcvQwVfyj3zy8O2+aM2PYos3c5pKQ05X/RJ +6bPl6taxcEsHQByNC0+7JJ1yPxYlqW28uDao4XNkrwhSkhzv0DylAGZNIrwAFkKq +dXRdCBMijvBB7jKvfPKK0aOlVJ7fdRo6PPoAJaDhmfsd5lbIGFcpwu4Rf2AiJQq3 +7ZzfGSUH/uvTsXW/e+QOQkr8cI4apRjGAuTImoIebZgkU8U9NOjCIwtBZO+KgiRZ +4cjC4mXgxMAdmmIMXUqy0QGiqgK2IGhpDneC9y2Al1WT/7Sz5au6tvMEIf5yrYuW +62/LyErTfNiwFGa4gYQ0nQ21ifwrA5bFFCyf4K0NyFRZaKJd/gpnszZtuw9NKxLn +5Lz10bI/aFPcOyKxgX0DqYHhiJFw0v4uTUIQK7RkDCQ+xAiJJ7c53/piCaqc6+IY +BRHvKTRK2jIxSlL8In+MxL+hDPXBm1c/NLIAqMogh7u0qYRg3U0V9leH5vZqatHI +/SDPyMAbcrjuIcg04fjaH2KW/REdPL87heoqH0tH5x0PnQqAUGxmuUm/7FEcoB+Q +oQOD/KIkZ5Abenmw/VJW11tt9A5dV6d3y+OBaTN2U1ZzT8PaWchjUimsIY3CVxTM +h5IxI1VTqMy9o/5mkA5ishzdUxh0hReO9NzUx4zgKFuWHAnUqYEkGC1okdSm+DqM +s3jtYwZcbXhV8USCZJWEyfV7T5/1iXR2/U432e7HN6Wv1uC/GQWafelkKosr2ulG +Y9Heehs56te3osz62G8Y27gCdZGi+GnysgegiLg7E2Qaep3UGk+Q3h8E+YAyQ0eK +H8gI6sKRjdIAGuhs7w== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java index 747ac86c881b..7b23839ce083 100644 --- a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java +++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java @@ -41,7 +41,6 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; @@ -124,6 +123,10 @@ public static Read read() { return new AutoValue_SolrIO_Read.Builder().setBatchSize(1000).setQuery("*:*").build(); } + public static ReadAll readAll() { + return new ReadAll(); + } + public static Write write() { // 1000 for batch size is good enough in many cases, // ex: if document size is large, around 10KB, the request's size will be around 10MB @@ -306,17 +309,22 @@ public abstract static class Read extends PTransform 0 && batchSize < MAX_BATCH_SIZE, - "Valid values for batchSize are 1 (inclusize) to %s (exclusive), but was: %s ", + "Valid values for batchSize are 1 (inclusive) to %s (exclusive), but was: %s ", MAX_BATCH_SIZE, batchSize); return builder().setBatchSize(batchSize).build(); } + /** Read from a specific Replica (partition). */ + public Read withReplicaInfo(ReplicaInfo replicaInfo) { + checkArgument(replicaInfo != null, "replicaInfo can not be null"); + return builder().setReplicaInfo(replicaInfo).build(); + } + @Override public PCollection expand(PBegin input) { checkArgument( getConnectionConfiguration() != null, "withConnectionConfiguration() is required"); checkArgument(getCollection() != null, "from() is required"); - - return input - .apply("Create", Create.of(this)) - .apply("Split", ParDo.of(new SplitFn())) - .apply("Reshuffle", Reshuffle.viaRandomKey()) - .apply("Read", ParDo.of(new ReadFn())); + return input.apply("Create", Create.of(this)).apply("ReadAll", readAll()); } @Override public void populateDisplayData(DisplayData.Builder builder) { super.populateDisplayData(builder); - builder.addIfNotNull(DisplayData.item("query", getQuery())); getConnectionConfiguration().populateDisplayData(builder); + builder.add(DisplayData.item("collection", getCollection())); + builder.addIfNotNull(DisplayData.item("query", getQuery())); + builder.add(DisplayData.item("batchSize", getBatchSize())); + final String replicaInfo = (getReplicaInfo() != null) ? getReplicaInfo().toString() : null; + builder.addIfNotNull(DisplayData.item("replicaInfo", replicaInfo)); } } /** A POJO describing a replica of Solr. */ @AutoValue - abstract static class ReplicaInfo implements Serializable { + public abstract static class ReplicaInfo implements Serializable { public abstract String coreName(); public abstract String coreUrl(); @@ -408,10 +421,9 @@ static ReplicaInfo create(Replica replica) { } } - static class SplitFn extends DoFn> { + private static class SplitFn extends DoFn { @ProcessElement - public void process(@Element SolrIO.Read spec, OutputReceiver> out) - throws IOException { + public void process(@Element Read spec, OutputReceiver out) throws IOException { ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration(); try (AuthorizedSolrClient client = connectionConfig.createClient()) { String collection = spec.getCollection(); @@ -437,19 +449,17 @@ public void process(@Element SolrIO.Read spec, OutputReceiver, SolrDocument> { + private static class ReadFn extends DoFn { @ProcessElement - public void process( - @Element KV specAndReplica, OutputReceiver out) - throws IOException { - Read spec = specAndReplica.getKey(); - ReplicaInfo replica = specAndReplica.getValue(); + public void process(@Element Read spec, OutputReceiver out) throws IOException { + ReplicaInfo replicaInfo = spec.getReplicaInfo(); + checkArgument(replicaInfo != null, "replicaInfo is required"); String cursorMark = CursorMarkParams.CURSOR_MARK_START; String query = spec.getQuery(); if (query == null) { @@ -459,7 +469,7 @@ public void process( solrQuery.setRows(spec.getBatchSize()); solrQuery.setDistrib(false); try (AuthorizedSolrClient client = - spec.getConnectionConfiguration().createClient(replica.baseUrl())) { + spec.getConnectionConfiguration().createClient(replicaInfo.baseUrl())) { SchemaRequest.UniqueKey request = new SchemaRequest.UniqueKey(); try { SchemaResponse.UniqueKeyResponse response = client.process(spec.getCollection(), request); @@ -472,7 +482,7 @@ public void process( solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); try { QueryResponse response; - response = client.query(replica.coreName(), solrQuery); + response = client.query(replicaInfo.coreName(), solrQuery); if (cursorMark.equals(response.getNextCursorMark())) { break; } @@ -488,6 +498,16 @@ public void process( } } + public static class ReadAll extends PTransform, PCollection> { + @Override + public PCollection expand(PCollection input) { + return input + .apply("Split", ParDo.of(new SplitFn())) + .apply("Reshuffle", Reshuffle.viaRandomKey()) + .apply("Read", ParDo.of(new ReadFn())); + } + } + /** A {@link PTransform} writing data to Solr. */ @AutoValue public abstract static class Write extends PTransform, PDone> { @@ -599,7 +619,7 @@ static class WriteFn extends DoFn { } @Setup - public void setup() throws Exception { + public void setup() { solrClient = spec.getConnectionConfiguration().createClient(); retryBackoff = diff --git a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java index 09c38a2b57e6..6f666b8600db 100644 --- a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java +++ b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java @@ -155,6 +155,23 @@ public void testRead() throws Exception { pipeline.run(); } + @Test + public void testReadAll() throws Exception { + SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient); + + PCollection output = + pipeline + .apply( + Create.of( + SolrIO.read() + .withConnectionConfiguration(connectionConfiguration) + .from(SOLR_COLLECTION) + .withBatchSize(101))) + .apply(SolrIO.readAll()); + PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo(NUM_DOCS); + pipeline.run(); + } + @Test public void testReadWithQuery() throws Exception { SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient); diff --git a/sdks/python/apache_beam/coders/row_coder.py b/sdks/python/apache_beam/coders/row_coder.py index 7012a56cb374..8c41891d0d8a 100644 --- a/sdks/python/apache_beam/coders/row_coder.py +++ b/sdks/python/apache_beam/coders/row_coder.py @@ -67,9 +67,6 @@ def is_deterministic(self): def to_type_hint(self): return named_tuple_from_schema(self.schema) - def as_cloud_object(self, coders_context=None): - raise NotImplementedError("as_cloud_object not supported for RowCoder") - def __hash__(self): return hash(self.schema.SerializeToString()) @@ -106,6 +103,8 @@ def coder_from_type(field_type): elif type_info == "array_type": return IterableCoder( RowCoder.coder_from_type(field_type.array_type.element_type)) + elif type_info == "row_type": + return RowCoder(field_type.row_type.schema) # The Java SDK supports several more types, but the coders are not yet # standard, and are not implemented in Python. diff --git a/sdks/python/apache_beam/coders/row_coder_test.py b/sdks/python/apache_beam/coders/row_coder_test.py index 0ffd98338190..8eb7ee26dcfb 100644 --- a/sdks/python/apache_beam/coders/row_coder_test.py +++ b/sdks/python/apache_beam/coders/row_coder_test.py @@ -49,9 +49,9 @@ class RowCoderTest(unittest.TestCase): - TEST_CASE = Person("Jon Snow", 23, None, ["crow", "wildling"]) - TEST_CASES = [ - TEST_CASE, + JON_SNOW = Person("Jon Snow", 23, None, ["crow", "wildling"]) + PEOPLE = [ + JON_SNOW, Person("Daenerys Targaryen", 25, "Westeros", ["Mother of Dragons"]), Person("Michael Bluth", 30, None, []) ] @@ -60,7 +60,7 @@ def test_create_row_coder_from_named_tuple(self): expected_coder = RowCoder(typing_to_runner_api(Person).row_type.schema) real_coder = coders_registry.get_coder(Person) - for test_case in self.TEST_CASES: + for test_case in self.PEOPLE: self.assertEqual( expected_coder.encode(test_case), real_coder.encode(test_case)) @@ -90,7 +90,7 @@ def test_create_row_coder_from_schema(self): ]) coder = RowCoder(schema) - for test_case in self.TEST_CASES: + for test_case in self.PEOPLE: self.assertEqual(test_case, coder.decode(coder.encode(test_case))) @unittest.skip( @@ -182,9 +182,17 @@ def test_row_coder_in_pipeine(self): with TestPipeline() as p: res = ( p - | beam.Create(self.TEST_CASES) + | beam.Create(self.PEOPLE) | beam.Filter(lambda person: person.name == "Jon Snow")) - assert_that(res, equal_to([self.TEST_CASE])) + assert_that(res, equal_to([self.JON_SNOW])) + + def test_row_coder_nested_struct(self): + Pair = typing.NamedTuple('Pair', [('left', Person), ('right', Person)]) + + value = Pair(self.PEOPLE[0], self.PEOPLE[1]) + coder = RowCoder(typing_to_runner_api(Pair).row_type.schema) + + self.assertEqual(value, coder.decode(coder.encode(value))) if __name__ == "__main__": diff --git a/sdks/python/apache_beam/dataframe/convert.py b/sdks/python/apache_beam/dataframe/convert.py index c55fa9efa20c..f0c5824b27f1 100644 --- a/sdks/python/apache_beam/dataframe/convert.py +++ b/sdks/python/apache_beam/dataframe/convert.py @@ -17,12 +17,21 @@ from __future__ import absolute_import import inspect +from typing import TYPE_CHECKING +from typing import Any +from typing import Dict +from typing import Tuple +from typing import Union from apache_beam import pvalue from apache_beam.dataframe import expressions from apache_beam.dataframe import frame_base from apache_beam.dataframe import transforms +if TYPE_CHECKING: + # pylint: disable=ungrouped-imports + import pandas + # TODO: Or should this be called as_dataframe? def to_dataframe( @@ -49,9 +58,9 @@ def to_dataframe( # TODO: Or should this be called from_dataframe? def to_pcollection( - *dataframes, # type: Tuple[frame_base.DeferredFrame] + *dataframes, # type: frame_base.DeferredFrame **kwargs): - # type: (...) -> Union[pvalue.PCollection, Tuple[pvalue.PCollection]] + # type: (...) -> Union[pvalue.PCollection, Tuple[pvalue.PCollection, ...]] """Converts one or more deferred dataframe-like objects back to a PCollection. @@ -67,18 +76,23 @@ def to_pcollection( if label is None: # Attempt to come up with a reasonable, stable label by retrieving the name # of these variables in the calling context. - previous_frame = inspect.currentframe().f_back + current_frame = inspect.currentframe() + if current_frame is None: + label = 'ToDataframe(...)' + + else: + previous_frame = current_frame.f_back - def name(obj): - for key, value in previous_frame.f_locals.items(): - if obj is value: - return key - for key, value in previous_frame.f_globals.items(): - if obj is value: - return key - return '...' + def name(obj): + for key, value in previous_frame.f_locals.items(): + if obj is value: + return key + for key, value in previous_frame.f_globals.items(): + if obj is value: + return key + return '...' - label = 'ToDataframe(%s)' % ', '.join(name(e) for e in dataframes) + label = 'ToDataframe(%s)' % ', '.join(name(e) for e in dataframes) def extract_input(placeholder): if not isinstance(placeholder._reference, pvalue.PCollection): @@ -91,7 +105,8 @@ def extract_input(placeholder): results = {p: extract_input(p) for p in placeholders } | label >> transforms._DataframeExpressionsTransform( - dict((ix, df._expr) for ix, df in enumerate(dataframes))) + dict((ix, df._expr) for ix, df in enumerate( + dataframes))) # type: Dict[Any, pvalue.PCollection] if len(results) == 1 and not always_return_tuple: return results[0] else: diff --git a/sdks/python/apache_beam/dataframe/doctests.py b/sdks/python/apache_beam/dataframe/doctests.py index a81494438037..030a58d101e7 100644 --- a/sdks/python/apache_beam/dataframe/doctests.py +++ b/sdks/python/apache_beam/dataframe/doctests.py @@ -43,6 +43,9 @@ import contextlib import doctest import re +from typing import Any +from typing import Dict +from typing import List import numpy as np import pandas as pd @@ -136,7 +139,7 @@ class _InMemoryResultRecorder(object): """ # Class-level value to survive pickling. - _ALL_RESULTS = {} + _ALL_RESULTS = {} # type: Dict[str, List[Any]] def __init__(self): self._id = id(self) diff --git a/sdks/python/apache_beam/dataframe/frame_base.py b/sdks/python/apache_beam/dataframe/frame_base.py index 275d2b99e275..61b399a8811a 100644 --- a/sdks/python/apache_beam/dataframe/frame_base.py +++ b/sdks/python/apache_beam/dataframe/frame_base.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import inspect +from typing import Dict import pandas as pd @@ -25,7 +26,7 @@ class DeferredFrame(object): - _pandas_type_map = {} + _pandas_type_map = {} # type: Dict[type, type] def __init__(self, expr): self._expr = expr diff --git a/sdks/python/apache_beam/dataframe/transforms.py b/sdks/python/apache_beam/dataframe/transforms.py index 31b62f1405fb..8c8943d73245 100644 --- a/sdks/python/apache_beam/dataframe/transforms.py +++ b/sdks/python/apache_beam/dataframe/transforms.py @@ -16,6 +16,15 @@ from __future__ import absolute_import +from typing import TYPE_CHECKING +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Tuple +from typing import TypeVar +from typing import Union + import pandas as pd import apache_beam as beam @@ -23,6 +32,12 @@ from apache_beam.dataframe import expressions from apache_beam.dataframe import frames # pylint: disable=unused-import +if TYPE_CHECKING: + # pylint: disable=ungrouped-imports + from apache_beam.pvalue import PCollection + +T = TypeVar('T') + class DataframeTransform(transforms.PTransform): """A PTransform for applying function that takes and returns dataframes @@ -82,8 +97,8 @@ def expand(self, inputs): def _apply_deferred_ops( self, - inputs, # type: Dict[PlaceholderExpr, PCollection] - outputs, # type: Dict[Any, Expression] + inputs, # type: Dict[expressions.Expression, PCollection] + outputs, # type: Dict[Any, expressions.Expression] ): # -> Dict[Any, PCollection] """Construct a Beam graph that evaluates a set of expressions on a set of input PCollections. @@ -248,7 +263,12 @@ def _dict_union(dicts): return result -def _flatten(valueish, root=()): +def _flatten( + valueish, # type: Union[T, List[T], Tuple[T], Dict[Any, T]] + root=(), # type: Tuple[Any, ...] + ): + # type: (...) -> Mapping[Tuple[Any, ...], T] + """Given a nested structure of dicts, tuples, and lists, return a flat dictionary where the values are the leafs and the keys are the "paths" to these leaves. diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index c2ca40117021..29d7577b9270 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -37,6 +37,7 @@ import argparse import base64 +import json from builtins import object from builtins import range from decimal import Decimal @@ -51,6 +52,13 @@ from apache_beam.testing.util import equal_to from apache_beam.transforms.core import PTransform +# Protect against environments where Google Cloud Natural Language client is +# not available. +try: + from apache_beam.ml.gcp import naturallanguageml as nlp +except ImportError: + nlp = None + # Quiet some pylint warnings that happen because of the somewhat special # format for the code snippets. # pylint:disable=invalid-name @@ -1520,3 +1528,86 @@ def bigqueryio_deadletter(): # [END BigQueryIODeadLetter] return result + + +def extract_sentiments(response): + # [START nlp_extract_sentiments] + return { + 'sentences': [{ + sentence.text.content: sentence.sentiment.score + } for sentence in response.sentences], + 'document_sentiment': response.document_sentiment.score, + } + # [END nlp_extract_sentiments] + + +def extract_entities(response): + # [START nlp_extract_entities] + return [{ + 'name': entity.name, + 'type': nlp.enums.Entity.Type(entity.type).name, + } for entity in response.entities] + # [END nlp_extract_entities] + + +def analyze_dependency_tree(response): + # [START analyze_dependency_tree] + from collections import defaultdict + adjacency_lists = [] + + index = 0 + for sentence in response.sentences: + adjacency_list = defaultdict(list) + sentence_begin = sentence.text.begin_offset + sentence_end = sentence_begin + len(sentence.text.content) - 1 + + while index < len(response.tokens) and \ + response.tokens[index].text.begin_offset <= sentence_end: + token = response.tokens[index] + head_token_index = token.dependency_edge.head_token_index + head_token_text = response.tokens[head_token_index].text.content + adjacency_list[head_token_text].append(token.text.content) + index += 1 + adjacency_lists.append(adjacency_list) + # [END analyze_dependency_tree] + + return adjacency_lists + + +def nlp_analyze_text(): + # [START nlp_analyze_text] + features = nlp.types.AnnotateTextRequest.Features( + extract_entities=True, + extract_document_sentiment=True, + extract_entity_sentiment=True, + extract_syntax=True, + ) + + with beam.Pipeline() as p: + responses = ( + p + | beam.Create([ + 'My experience so far has been fantastic! ' + 'I\'d really recommend this product.' + ]) + | beam.Map(lambda x: nlp.Document(x, type='PLAIN_TEXT')) + | nlp.AnnotateText(features)) + + _ = ( + responses + | beam.Map(extract_sentiments) + | 'Parse sentiments to JSON' >> beam.Map(json.dumps) + | 'Write sentiments' >> beam.io.WriteToText('sentiments.txt')) + + _ = ( + responses + | beam.Map(extract_entities) + | 'Parse entities to JSON' >> beam.Map(json.dumps) + | 'Write entities' >> beam.io.WriteToText('entities.txt')) + + _ = ( + responses + | beam.Map(analyze_dependency_tree) + | 'Parse adjacency list to JSON' >> beam.Map(json.dumps) + | 'Write adjacency list' >> beam.io.WriteToText('adjancency_list.txt')) + # [END nlp_analyze_text] diff --git a/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py new file mode 100644 index 000000000000..7dbbf837748d --- /dev/null +++ b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py @@ -0,0 +1,141 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Integration test for Python cross-language pipelines for Java KafkaIO.""" + +from __future__ import absolute_import + +import contextlib +import logging +import os +import socket +import subprocess +import time +import typing +import unittest + +import apache_beam as beam +from apache_beam.io.external.kafka import ReadFromKafka +from apache_beam.io.external.kafka import WriteToKafka +from apache_beam.metrics import Metrics +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.testing.test_pipeline import TestPipeline + + +class CrossLanguageKafkaIO(object): + def __init__(self, bootstrap_servers, topic, expansion_service=None): + self.bootstrap_servers = bootstrap_servers + self.topic = topic + self.expansion_service = expansion_service + self.sum_counter = Metrics.counter('source', 'elements_sum') + + def build_write_pipeline(self, pipeline): + _ = ( + pipeline + | 'Impulse' >> beam.Impulse() + | 'Generate' >> beam.FlatMap(lambda x: range(1000)) # pylint: disable=range-builtin-not-iterating + | 'Reshuffle' >> beam.Reshuffle() + | 'MakeKV' >> beam.Map(lambda x: + (b'', str(x).encode())).with_output_types( + typing.Tuple[bytes, bytes]) + | 'WriteToKafka' >> WriteToKafka( + producer_config={'bootstrap.servers': self.bootstrap_servers}, + topic=self.topic, + expansion_service=self.expansion_service)) + + def build_read_pipeline(self, pipeline): + _ = ( + pipeline + | 'ReadFromKafka' >> ReadFromKafka( + consumer_config={ + 'bootstrap.servers': self.bootstrap_servers, + 'auto.offset.reset': 'earliest' + }, + topics=[self.topic], + expansion_service=self.expansion_service) + | 'Windowing' >> beam.WindowInto( + beam.window.FixedWindows(300), + trigger=beam.transforms.trigger.AfterProcessingTime(60), + accumulation_mode=beam.transforms.trigger.AccumulationMode. + DISCARDING) + | 'DecodingValue' >> beam.Map(lambda elem: int(elem[1].decode())) + | 'CombineGlobally' >> beam.CombineGlobally(sum).without_defaults() + | 'SetSumCounter' >> beam.Map(self.sum_counter.inc)) + + def run_xlang_kafkaio(self, pipeline): + self.build_write_pipeline(pipeline) + self.build_read_pipeline(pipeline) + pipeline.run(False) + + +@unittest.skipUnless( + os.environ.get('LOCAL_KAFKA_JAR'), + "LOCAL_KAFKA_JAR environment var is not provided.") +class CrossLanguageKafkaIOTest(unittest.TestCase): + def get_open_port(self): + s = None + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except: # pylint: disable=bare-except + # Above call will fail for nodes that only support IPv6. + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + s.bind(('localhost', 0)) + s.listen(1) + port = s.getsockname()[1] + s.close() + return port + + @contextlib.contextmanager + def local_kafka_service(self, local_kafka_jar_file): + kafka_port = str(self.get_open_port()) + zookeeper_port = str(self.get_open_port()) + kafka_server = None + try: + kafka_server = subprocess.Popen( + ['java', '-jar', local_kafka_jar_file, kafka_port, zookeeper_port]) + time.sleep(3) + yield kafka_port + finally: + if kafka_server: + kafka_server.kill() + + def get_options(self): + options = PipelineOptions([ + '--runner', + 'FlinkRunner', + '--parallelism', + '2', + '--experiment', + 'beam_fn_api' + ]) + return options + + def test_kafkaio_write(self): + local_kafka_jar = os.environ.get('LOCAL_KAFKA_JAR') + with self.local_kafka_service(local_kafka_jar) as kafka_port: + options = self.get_options() + p = TestPipeline(options=options) + p.not_use_test_runner_api = True + CrossLanguageKafkaIO('localhost:%s' % kafka_port, + 'xlang_kafkaio_test').build_write_pipeline(p) + job = p.run() + job.wait_until_finish() + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 130b473300f3..174edec2848e 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -271,6 +271,8 @@ def compute_table_name(row): from apache_beam.transforms import ParDo from apache_beam.transforms import PTransform from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.sideinputs import SIDE_INPUT_PREFIX +from apache_beam.transforms.sideinputs import get_sideinput_index from apache_beam.transforms.window import GlobalWindows from apache_beam.utils import retry from apache_beam.utils.annotations import deprecated @@ -1396,6 +1398,9 @@ def __init__( and https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json. """ + self._table = table + self._dataset = dataset + self._project = project self.table_reference = bigquery_tools.parse_table_reference( table, dataset, project) self.create_disposition = BigQueryDisposition.validate_create( @@ -1523,6 +1528,73 @@ def display_data(self): res['table'] = DisplayDataItem(tableSpec, label='Table') return res + def to_runner_api_parameter(self, context): + from apache_beam.internal import pickler + + # It'd be nice to name these according to their actual + # names/positions in the orignal argument list, but such a + # transformation is currently irreversible given how + # remove_objects_from_args and insert_values_in_args + # are currently implemented. + def serialize(side_inputs): + return {(SIDE_INPUT_PREFIX + '%s') % ix: + si.to_runner_api(context).SerializeToString() + for ix, + si in enumerate(side_inputs)} + + table_side_inputs = serialize(self.table_side_inputs) + schema_side_inputs = serialize(self.schema_side_inputs) + + config = { + 'table': self._table, + 'dataset': self._dataset, + 'project': self._project, + 'schema': self.schema, + 'create_disposition': self.create_disposition, + 'write_disposition': self.write_disposition, + 'kms_key': self.kms_key, + 'batch_size': self.batch_size, + 'max_file_size': self.max_file_size, + 'max_files_per_bundle': self.max_files_per_bundle, + 'custom_gcs_temp_location': self.custom_gcs_temp_location, + 'method': self.method, + 'insert_retry_strategy': self.insert_retry_strategy, + 'additional_bq_parameters': self.additional_bq_parameters, + 'table_side_inputs': table_side_inputs, + 'schema_side_inputs': schema_side_inputs, + 'triggering_frequency': self.triggering_frequency, + 'validate': self._validate, + 'temp_file_format': self._temp_file_format, + } + return 'beam:transform:write_to_big_query:v0', pickler.dumps(config) + + @PTransform.register_urn('beam:transform:write_to_big_query:v0', bytes) + def from_runner_api(unused_ptransform, payload, context): + from apache_beam.internal import pickler + from apache_beam.portability.api.beam_runner_api_pb2 import SideInput + + config = pickler.loads(payload) + + def deserialize(side_inputs): + deserialized_side_inputs = {} + for k, v in side_inputs.items(): + side_input = SideInput() + side_input.ParseFromString(v) + deserialized_side_inputs[k] = side_input + + # This is an ordered list stored as a dict (see the comments in + # to_runner_api_parameter above). + indexed_side_inputs = [( + get_sideinput_index(tag), + pvalue.AsSideInput.from_runner_api(si, context)) for tag, + si in deserialized_side_inputs.items()] + return [si for _, si in sorted(indexed_side_inputs)] + + config['table_side_inputs'] = deserialize(config['table_side_inputs']) + config['schema_side_inputs'] = deserialize(config['schema_side_inputs']) + + return WriteToBigQuery(**config) + class _PassThroughThenCleanup(PTransform): """A PTransform that invokes a DoFn after the input PCollection has been diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index 8c2bfe8f0d76..5c0597869517 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -584,6 +584,73 @@ def test_schema_autodetect_not_allowed_with_avro_file_loads(self): schema=beam.io.gcp.bigquery.SCHEMA_AUTODETECT, temp_file_format=bigquery_tools.FileFormat.AVRO)) + def test_to_from_runner_api(self): + """Tests that serialization of WriteToBigQuery is correct. + + This is not intended to be a change-detector test. As such, this only tests + the more complicated serialization logic of parameters: ValueProviders, + callables, and side inputs. + """ + FULL_OUTPUT_TABLE = 'test_project:output_table' + + p = TestPipeline( + additional_pipeline_args=["--experiments=use_beam_bq_sink"]) + + # Used for testing side input parameters. + table_record_pcv = beam.pvalue.AsDict( + p | "MakeTable" >> beam.Create([('table', FULL_OUTPUT_TABLE)])) + + # Used for testing value provider parameters. + schema = value_provider.StaticValueProvider(str, '"a:str"') + + original = WriteToBigQuery( + table=lambda _, + side_input: side_input['table'], + table_side_inputs=(table_record_pcv, ), + schema=schema) + + # pylint: disable=expression-not-assigned + p | 'MyWriteToBigQuery' >> original + + # Run the pipeline through to generate a pipeline proto from an empty + # context. This ensures that the serialization code ran. + pipeline_proto, context = TestPipeline.from_runner_api( + p.to_runner_api(), p.runner, p.get_pipeline_options()).to_runner_api( + return_context=True) + + # Find the transform from the context. + write_to_bq_id = [ + k for k, + v in pipeline_proto.components.transforms.items() + if v.unique_name == 'MyWriteToBigQuery' + ][0] + deserialized_node = context.transforms.get_by_id(write_to_bq_id) + deserialized = deserialized_node.transform + self.assertIsInstance(deserialized, WriteToBigQuery) + + # Test that the serialization of a value provider is correct. + self.assertEqual(original.schema, deserialized.schema) + + # Test that the serialization of a callable is correct. + self.assertEqual( + deserialized._table(None, {'table': FULL_OUTPUT_TABLE}), + FULL_OUTPUT_TABLE) + + # Test that the serialization of a side input is correct. + self.assertEqual( + len(original.table_side_inputs), len(deserialized.table_side_inputs)) + original_side_input_data = original.table_side_inputs[0]._side_input_data() + deserialized_side_input_data = deserialized.table_side_inputs[ + 0]._side_input_data() + self.assertEqual( + original_side_input_data.access_pattern, + deserialized_side_input_data.access_pattern) + self.assertEqual( + original_side_input_data.window_mapping_fn, + deserialized_side_input_data.window_mapping_fn) + self.assertEqual( + original_side_input_data.view_fn, deserialized_side_input_data.view_fn) + @unittest.skipIf(HttpError is None, 'GCP dependencies are not installed') class BigQueryStreamingInsertTransformTests(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/hadoopfilesystem_test.py b/sdks/python/apache_beam/io/hadoopfilesystem_test.py index a4b513b5876b..e83091f145c9 100644 --- a/sdks/python/apache_beam/io/hadoopfilesystem_test.py +++ b/sdks/python/apache_beam/io/hadoopfilesystem_test.py @@ -190,12 +190,15 @@ def rename(self, path1, path2): if self.status(path1, strict=False) is None: raise FakeHdfsError('Path1 not found: %s' % path1) - for fullpath in self.files.keys(): # pylint: disable=consider-iterating-dictionary - if fullpath == path1 or fullpath.startswith(path1 + '/'): - f = self.files.pop(fullpath) - newpath = path2 + fullpath[len(path1):] - f.stat['path'] = newpath - self.files[newpath] = f + files_to_rename = [ + path for path in self.files + if path == path1 or path.startswith(path1 + '/') + ] + for fullpath in files_to_rename: + f = self.files.pop(fullpath) + newpath = path2 + fullpath[len(path1):] + f.stat['path'] = newpath + self.files[newpath] = f def checksum(self, path): f = self.files.get(path, None) diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py index 4099e87215e7..0660b8747f3a 100644 --- a/sdks/python/apache_beam/io/textio.py +++ b/sdks/python/apache_beam/io/textio.py @@ -438,7 +438,7 @@ def _create_text_source( class ReadAllFromText(PTransform): """A ``PTransform`` for reading a ``PCollection`` of text files. - Reads a ``PCollection`` of text files or file patterns and and produces a + Reads a ``PCollection`` of text files or file patterns and produces a ``PCollection`` of strings. Parses a text file as newline-delimited elements, by default assuming diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py index f411549d7b9c..397256e5f153 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test_it.py @@ -44,7 +44,7 @@ def extract_entities_descriptions(response): for result in response.annotation_results: - for segment in result.segment_label_annotations: + for segment in result.segment_presence_label_annotations: yield segment.entity.description @@ -69,7 +69,9 @@ def test_label_detection_with_video_context(self): | beam.ParDo(extract_entities_descriptions) | beam.combiners.ToList()) - assert_that(output, matches_all([hc.has_items('bicycle', 'dinosaur')])) + # Search for at least one entity that contains 'bicycle'. + assert_that( + output, matches_all([hc.has_item(hc.contains_string('bicycle'))])) if __name__ == '__main__': diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index d1380c6294b7..9d1b02c53e01 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -509,7 +509,8 @@ def run(self, test_runner_api='AUTO'): return Pipeline.from_runner_api( self.to_runner_api(use_fake_coders=True), self.runner, - self._options).run(False) + self._options, + allow_proto_holders=True).run(False) if self._options.view_as(TypeOptions).runtime_type_check: from apache_beam.typehints import typecheck diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 0011109f5d25..a7e0eac9c860 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -27,7 +27,6 @@ from builtins import object from builtins import range -import mock from nose.plugins.attrib import attr import apache_beam as beam @@ -370,9 +369,7 @@ def raise_exception(exn): # pylint: disable=expression-not-assigned p | Create([ValueError('msg')]) | Map(raise_exception) - @mock.patch( - 'apache_beam.runners.direct.direct_runner._get_transform_overrides') - def test_ptransform_overrides(self, file_system_override_mock): + def test_ptransform_overrides(self): class MyParDoOverride(PTransformOverride): def matches(self, applied_ptransform): return isinstance(applied_ptransform.transform, DoubleParDo) @@ -382,15 +379,12 @@ def get_replacement_transform(self, ptransform): return TripleParDo() raise ValueError('Unsupported type of transform: %r' % ptransform) - def get_overrides(unused_pipeline_options): - return [MyParDoOverride()] + p = Pipeline() + pcoll = p | beam.Create([1, 2, 3]) | 'Multiply' >> DoubleParDo() + assert_that(pcoll, equal_to([3, 6, 9])) - file_system_override_mock.side_effect = get_overrides - - # Specify DirectRunner as it's the one patched above. - with Pipeline(runner='BundleBasedDirectRunner') as p: - pcoll = p | beam.Create([1, 2, 3]) | 'Multiply' >> DoubleParDo() - assert_that(pcoll, equal_to([3, 6, 9])) + p.replace_all([MyParDoOverride()]) + p.run() def test_ptransform_override_type_hints(self): class NoTypeHintOverride(PTransformOverride): diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index ae478da786a8..063dd6ec893a 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -26,7 +26,6 @@ from __future__ import division import base64 -import json import logging import os import subprocess @@ -278,7 +277,7 @@ def group_by_key_input_visitor(): class GroupByKeyInputVisitor(PipelineVisitor): """A visitor that replaces `Any` element type for input `PCollection` of - a `GroupByKey` or `_GroupByKeyOnly` with a `KV` type. + a `GroupByKey` with a `KV` type. TODO(BEAM-115): Once Python SDk is compatible with the new Runner API, we could directly replace the coder instead of mutating the element type. @@ -289,8 +288,8 @@ def enter_composite_transform(self, transform_node): def visit_transform(self, transform_node): # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.transforms.core import GroupByKey, _GroupByKeyOnly - if isinstance(transform_node.transform, (GroupByKey, _GroupByKeyOnly)): + from apache_beam.transforms.core import GroupByKey + if isinstance(transform_node.transform, GroupByKey): pcoll = transform_node.inputs[0] pcoll.element_type = typehints.coerce_to_kv_type( pcoll.element_type, transform_node.full_label) @@ -464,6 +463,9 @@ def run_pipeline(self, pipeline, options): # any added PTransforms. pipeline.replace_all(DataflowRunner._PTRANSFORM_OVERRIDES) + from apache_beam.runners.dataflow.ptransform_overrides import WriteToBigQueryPTransformOverride + pipeline.replace_all([WriteToBigQueryPTransformOverride(pipeline, options)]) + if (apiclient._use_fnapi(options) and not apiclient._use_unified_worker(options)): pipeline.replace_all(DataflowRunner._JRH_PTRANSFORM_OVERRIDES) @@ -805,49 +807,24 @@ def run_Flatten(self, transform_node, options): PropertyNames.OUTPUT_NAME: PropertyNames.OUT }]) - def apply_WriteToBigQuery(self, transform, pcoll, options): - # Make sure this is the WriteToBigQuery class that we expected, and that - # users did not specifically request the new BQ sink by passing experiment - # flag. - - # TODO(BEAM-6928): Remove this function for release 2.14.0. - experiments = options.view_as(DebugOptions).experiments or [] - from apache_beam.runners.dataflow.internal import apiclient - use_fnapi = apiclient._use_fnapi(options) - if (not isinstance(transform, beam.io.WriteToBigQuery) or use_fnapi or - 'use_beam_bq_sink' in experiments): - return self.apply_PTransform(transform, pcoll, options) - if transform.schema == beam.io.gcp.bigquery.SCHEMA_AUTODETECT: - raise RuntimeError( - 'Schema auto-detection is not supported on the native sink') - standard_options = options.view_as(StandardOptions) - if standard_options.streaming: - if (transform.write_disposition == - beam.io.BigQueryDisposition.WRITE_TRUNCATE): - raise RuntimeError('Can not use write truncation mode in streaming') - return self.apply_PTransform(transform, pcoll, options) - else: - from apache_beam.io.gcp.bigquery_tools import parse_table_schema_from_json - schema = None - if transform.schema: - schema = parse_table_schema_from_json(json.dumps(transform.schema)) - return pcoll | 'WriteToBigQuery' >> beam.io.Write( - beam.io.BigQuerySink( - transform.table_reference.tableId, - transform.table_reference.datasetId, - transform.table_reference.projectId, - schema, - transform.create_disposition, - transform.write_disposition, - kms_key=transform.kms_key)) - + # TODO(srohde): Remove this after internal usages have been removed. def apply_GroupByKey(self, transform, pcoll, options): + return transform.expand(pcoll) + + def _verify_gbk_coders(self, transform, pcoll): # Infer coder of parent. # # TODO(ccy): make Coder inference and checking less specialized and more # comprehensive. + parent = pcoll.producer if parent: + # Skip the check because we can assume that any x-lang transform is + # properly formed (the onus is on the expansion service to construct + # transforms correctly). + if isinstance(parent.transform, RunnerAPIPTransformHolder): + return + coder = parent.transform._infer_output_coder() # pylint: disable=protected-access if not coder: coder = self._get_coder(pcoll.element_type or typehints.Any, None) @@ -859,11 +836,13 @@ def apply_GroupByKey(self, transform, pcoll, options): coders.registry.verify_deterministic( coder.key_coder(), 'GroupByKey operation "%s"' % transform.label) - return pvalue.PCollection.from_(pcoll) - def run_GroupByKey(self, transform_node, options): input_tag = transform_node.inputs[0].tag input_step = self._cache.get_pvalue(transform_node.inputs[0]) + + # Verify that the GBK's parent has a KV coder. + self._verify_gbk_coders(transform_node.transform, transform_node.inputs[0]) + step = self._add_step( TransformNames.GROUP, transform_node.full_label, transform_node) step.add_property( diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index d94be78749a8..36e222d90b19 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -55,7 +55,6 @@ from apache_beam.transforms import environments from apache_beam.transforms import window from apache_beam.transforms.core import Windowing -from apache_beam.transforms.core import _GroupByKeyOnly from apache_beam.transforms.display import DisplayDataItem from apache_beam.typehints import typehints @@ -226,6 +225,9 @@ def test_environment_override_translation(self): capabilities=environments.python_sdk_capabilities()) ]) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_remote_runner_translation(self): remote_runner = DataflowRunner() with Pipeline(remote_runner, @@ -236,6 +238,9 @@ def test_remote_runner_translation(self): | 'Do' >> ptransform.FlatMap(lambda x: [(x, x)]) | ptransform.GroupByKey()) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_streaming_create_translation(self): remote_runner = DataflowRunner() self.default_properties.append("--streaming") @@ -251,6 +256,9 @@ def test_streaming_create_translation(self): self.assertEqual(job_dict[u'steps'][1][u'kind'], u'ParallelDo') self.assertEqual(job_dict[u'steps'][2][u'kind'], u'ParallelDo') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_bigquery_read_streaming_fail(self): remote_runner = DataflowRunner() self.default_properties.append("--streaming") @@ -260,6 +268,9 @@ def test_bigquery_read_streaming_fail(self): PipelineOptions(self.default_properties)) as p: _ = p | beam.io.Read(beam.io.BigQuerySource('some.table')) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_biqquery_read_fn_api_fail(self): remote_runner = DataflowRunner() for flag in ['beam_fn_api', 'use_unified_worker', 'use_runner_v2']: @@ -272,6 +283,9 @@ def test_biqquery_read_fn_api_fail(self): PipelineOptions(self.default_properties)) as p: _ = p | beam.io.Read(beam.io.BigQuerySource('some.table')) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_remote_runner_display_data(self): remote_runner = DataflowRunner() p = Pipeline( @@ -314,55 +328,51 @@ def test_remote_runner_display_data(self): }] self.assertUnhashableCountEqual(disp_data, expected_data) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_no_group_by_key_directly_after_bigquery(self): remote_runner = DataflowRunner() - p = Pipeline( - remote_runner, - options=PipelineOptions([ - '--dataflow_endpoint=ignored', - '--job_name=test-job', - '--project=test-project', - '--staging_location=ignored', - '--temp_location=/dev/null', - '--no_auth' - ])) - rows = p | beam.io.Read(beam.io.BigQuerySource('dataset.faketable')) with self.assertRaises(ValueError, msg=('Coder for the GroupByKey operation' '"GroupByKey" is not a key-value coder: ' 'RowAsDictJsonCoder')): - unused_invalid = rows | beam.GroupByKey() + with beam.Pipeline(runner=remote_runner, + options=PipelineOptions(self.default_properties)) as p: + # pylint: disable=expression-not-assigned + p | beam.io.Read( + beam.io.BigQuerySource('dataset.faketable')) | beam.GroupByKey() def test_group_by_key_input_visitor_with_valid_inputs(self): p = TestPipeline() pcoll1 = PCollection(p) pcoll2 = PCollection(p) pcoll3 = PCollection(p) - for transform in [_GroupByKeyOnly(), beam.GroupByKey()]: - pcoll1.element_type = None - pcoll2.element_type = typehints.Any - pcoll3.element_type = typehints.KV[typehints.Any, typehints.Any] - for pcoll in [pcoll1, pcoll2, pcoll3]: - applied = AppliedPTransform(None, transform, "label", [pcoll]) - applied.outputs[None] = PCollection(None) - DataflowRunner.group_by_key_input_visitor().visit_transform(applied) - self.assertEqual( - pcoll.element_type, typehints.KV[typehints.Any, typehints.Any]) + + pcoll1.element_type = None + pcoll2.element_type = typehints.Any + pcoll3.element_type = typehints.KV[typehints.Any, typehints.Any] + for pcoll in [pcoll1, pcoll2, pcoll3]: + applied = AppliedPTransform(None, beam.GroupByKey(), "label", [pcoll]) + applied.outputs[None] = PCollection(None) + DataflowRunner.group_by_key_input_visitor().visit_transform(applied) + self.assertEqual( + pcoll.element_type, typehints.KV[typehints.Any, typehints.Any]) def test_group_by_key_input_visitor_with_invalid_inputs(self): p = TestPipeline() pcoll1 = PCollection(p) pcoll2 = PCollection(p) - for transform in [_GroupByKeyOnly(), beam.GroupByKey()]: - pcoll1.element_type = str - pcoll2.element_type = typehints.Set - err_msg = ( - r"Input to 'label' must be compatible with KV\[Any, Any\]. " - "Found .*") - for pcoll in [pcoll1, pcoll2]: - with self.assertRaisesRegex(ValueError, err_msg): - DataflowRunner.group_by_key_input_visitor().visit_transform( - AppliedPTransform(None, transform, "label", [pcoll])) + + pcoll1.element_type = str + pcoll2.element_type = typehints.Set + err_msg = ( + r"Input to 'label' must be compatible with KV\[Any, Any\]. " + "Found .*") + for pcoll in [pcoll1, pcoll2]: + with self.assertRaisesRegex(ValueError, err_msg): + DataflowRunner.group_by_key_input_visitor().visit_transform( + AppliedPTransform(None, beam.GroupByKey(), "label", [pcoll])) def test_group_by_key_input_visitor_for_non_gbk_transforms(self): p = TestPipeline() @@ -446,6 +456,9 @@ def test_side_input_visitor(self): common_urns.side_inputs.MULTIMAP.urn, side_input._side_input_data().access_pattern) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_min_cpu_platform_flag_is_propagated_to_experiments(self): remote_runner = DataflowRunner() self.default_properties.append('--min_cpu_platform=Intel Haswell') @@ -456,6 +469,9 @@ def test_min_cpu_platform_flag_is_propagated_to_experiments(self): 'min_cpu_platform=Intel Haswell', remote_runner.job.options.view_as(DebugOptions).experiments) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_streaming_engine_flag_adds_windmill_experiments(self): remote_runner = DataflowRunner() self.default_properties.append('--streaming') @@ -471,6 +487,9 @@ def test_streaming_engine_flag_adds_windmill_experiments(self): self.assertIn('enable_windmill_service', experiments_for_job) self.assertIn('some_other_experiment', experiments_for_job) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_upload_graph_experiment(self): remote_runner = DataflowRunner() self.default_properties.append('--experiment=upload_graph') @@ -482,6 +501,9 @@ def test_upload_graph_experiment(self): remote_runner.job.options.view_as(DebugOptions).experiments) self.assertIn('upload_graph', experiments_for_job) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_dataflow_worker_jar_flag_non_fnapi_noop(self): remote_runner = DataflowRunner() self.default_properties.append('--experiment=some_other_experiment') @@ -495,6 +517,9 @@ def test_dataflow_worker_jar_flag_non_fnapi_noop(self): self.assertIn('some_other_experiment', experiments_for_job) self.assertNotIn('use_staged_dataflow_worker_jar', experiments_for_job) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_dataflow_worker_jar_flag_adds_use_staged_worker_jar_experiment(self): remote_runner = DataflowRunner() self.default_properties.append('--experiment=beam_fn_api') @@ -508,6 +533,9 @@ def test_dataflow_worker_jar_flag_adds_use_staged_worker_jar_experiment(self): self.assertIn('beam_fn_api', experiments_for_job) self.assertIn('use_staged_dataflow_worker_jar', experiments_for_job) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_use_fastavro_experiment_is_added_on_py3_and_onwards(self): remote_runner = DataflowRunner() @@ -519,6 +547,9 @@ def test_use_fastavro_experiment_is_added_on_py3_and_onwards(self): remote_runner.job.options.view_as(DebugOptions).lookup_experiment( 'use_fastavro', False)) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_use_fastavro_experiment_is_not_added_when_use_avro_is_present(self): remote_runner = DataflowRunner() self.default_properties.append('--experiment=use_avro') @@ -530,6 +561,9 @@ def test_use_fastavro_experiment_is_not_added_when_use_avro_is_present(self): self.assertFalse(debug_options.lookup_experiment('use_fastavro', False)) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_unsupported_fnapi_features(self): remote_runner = DataflowRunner() self.default_properties.append('--experiment=beam_fn_api') @@ -583,6 +617,9 @@ def test_get_default_gcp_region_ignores_error( result = runner.get_default_gcp_region() self.assertIsNone(result) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_combine_values_translation(self): runner = DataflowRunner() @@ -597,6 +634,15 @@ def test_combine_values_translation(self): self.assertIn( u'CombineValues', set(step[u'kind'] for step in job_dict[u'steps'])) + def _find_step(self, job, step_name): + job_dict = json.loads(str(job)) + maybe_step = [ + s for s in job_dict[u'steps'] + if s[u'properties'][u'user_name'] == step_name + ] + self.assertTrue(maybe_step, 'Could not find step {}'.format(step_name)) + return maybe_step[0] + def expect_correct_override(self, job, step_name, step_kind): """Expects that a transform was correctly overriden.""" @@ -616,14 +662,7 @@ def expect_correct_override(self, job, step_name, step_kind): "user_name": step_name + ".out" }] - job_dict = json.loads(str(job)) - maybe_step = [ - s for s in job_dict[u'steps'] - if s[u'properties'][u'user_name'] == step_name - ] - self.assertTrue(maybe_step, 'Could not find step {}'.format(step_name)) - - step = maybe_step[0] + step = self._find_step(job, step_name) self.assertEqual(step[u'kind'], step_kind) # The display data here is forwarded because the replace transform is @@ -631,6 +670,9 @@ def expect_correct_override(self, job, step_name, step_kind): self.assertGreater(len(step[u'properties']['display_data']), 0) self.assertEqual(step[u'properties']['output_info'], expected_output_info) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_read_create_translation(self): runner = DataflowRunner() @@ -641,6 +683,9 @@ def test_read_create_translation(self): self.expect_correct_override(runner.job, u'Create/Read', u'ParallelRead') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_read_bigquery_translation(self): runner = DataflowRunner() @@ -651,6 +696,9 @@ def test_read_bigquery_translation(self): self.expect_correct_override(runner.job, u'Read', u'ParallelRead') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_read_pubsub_translation(self): runner = DataflowRunner() @@ -664,6 +712,113 @@ def test_read_pubsub_translation(self): self.expect_correct_override( runner.job, u'ReadFromPubSub/Read', u'ParallelRead') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') + def test_gbk_translation(self): + runner = DataflowRunner() + with beam.Pipeline(runner=runner, + options=PipelineOptions(self.default_properties)) as p: + # pylint: disable=expression-not-assigned + p | beam.Create([(1, 2)]) | beam.GroupByKey() + + expected_output_info = [{ + "encoding": { + "@type": "kind:windowed_value", + "component_encodings": [{ + "@type": "kind:pair", + "component_encodings": [{ + "@type": "kind:varint" + }, + { + "@type": "kind:stream", + "component_encodings": [{ + "@type": "kind:varint" + }], + "is_stream_like": True + }], + "is_pair_like": True + }, { + "@type": "kind:global_window" + }], + "is_wrapper": True + }, + "output_name": "out", + "user_name": "GroupByKey.out" + }] # yapf: disable + + gbk_step = self._find_step(runner.job, u'GroupByKey') + self.assertEqual(gbk_step[u'kind'], u'GroupByKey') + self.assertEqual( + gbk_step[u'properties']['output_info'], expected_output_info) + + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') + def test_write_bigquery_translation(self): + runner = DataflowRunner() + + with beam.Pipeline(runner=runner, + options=PipelineOptions(self.default_properties)) as p: + # pylint: disable=expression-not-assigned + p | beam.Create([1]) | beam.io.WriteToBigQuery('some.table') + + job_dict = json.loads(str(runner.job)) + + expected_step = { + "kind": "ParallelWrite", + "name": "s2", + "properties": { + "create_disposition": "CREATE_IF_NEEDED", + "dataset": "some", + "display_data": [], + "encoding": { + "@type": "kind:windowed_value", + "component_encodings": [{ + "component_encodings": [], + "pipeline_proto_coder_id": "ref_Coder_RowAsDictJsonCoder_4" + }, { + "@type": "kind:global_window" + }], + "is_wrapper": True + }, + "format": "bigquery", + "parallel_input": { + "@type": "OutputReference", + "output_name": "out", + "step_name": "s1" + }, + "table": "table", + "user_name": "WriteToBigQuery/Write/NativeWrite", + "write_disposition": "WRITE_APPEND" + } + } + job_dict = json.loads(str(runner.job)) + write_step = [ + s for s in job_dict[u'steps'] + if s[u'properties'][u'user_name'].startswith('WriteToBigQuery') + ][0] + + # Delete the @type field because in this case it is a hash which may change + # depending on the pickling version. + step_encoding = write_step[u'properties'][u'encoding'] + del step_encoding[u'component_encodings'][0][u'@type'] + self.assertEqual(expected_step, write_step) + + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') + def test_write_bigquery_failed_translation(self): + """Tests that WriteToBigQuery cannot have any consumers if replaced.""" + runner = DataflowRunner() + + with self.assertRaises(ValueError): + with beam.Pipeline(runner=runner, + options=PipelineOptions(self.default_properties)) as p: + # pylint: disable=expression-not-assigned + out = p | beam.Create([1]) | beam.io.WriteToBigQuery('some.table') + out['FailedRows'] | 'MyTransform' >> beam.Map(lambda _: _) + class CustomMergingWindowFn(window.WindowFn): def assign(self, assign_context): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 06bb26029aeb..4fb7e6a9f2f3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -314,8 +314,15 @@ def __init__( if container_image_url in already_added_containers: # Do not add the pipeline environment again. + # Currently, Dataflow uses Docker container images to uniquely + # identify execution environments. Hence Dataflow executes all + # transforms that specify the the same Docker container image in a + # single container instance. Dependencies of all environments that + # specify a given container image will be staged in the container + # instance for that particular container image. # TODO(BEAM-9455): loosen this restriction to support multiple - # environments with the same container name. + # environments with the same container image when Dataflow supports + # environment specific artifact provisioning. continue already_added_containers.append(container_image_url) @@ -583,6 +590,7 @@ def _stage_resources(self, pipeline, options): raise RuntimeError('The --temp_location option must be specified.') resources = [] + hashs = {} for _, env in sorted(pipeline.components.environments.items(), key=lambda kv: kv[0]): for dep in env.dependencies: @@ -595,7 +603,16 @@ def _stage_resources(self, pipeline, options): role_payload = ( beam_runner_api_pb2.ArtifactStagingToRolePayload.FromString( dep.role_payload)) - resources.append((type_payload.path, role_payload.staged_name)) + if type_payload.sha256 and type_payload.sha256 in hashs: + _LOGGER.info( + 'Found duplicated artifact: %s (%s)', + type_payload.path, + type_payload.sha256) + dep.role_payload = beam_runner_api_pb2.ArtifactStagingToRolePayload( + staged_name=hashs[type_payload.sha256]).SerializeToString() + else: + resources.append((type_payload.path, role_payload.staged_name)) + hashs[type_payload.sha256] = role_payload.staged_name resource_stager = _LegacyDataflowStager(self) staged_resources = resource_stager.stage_job_resources( diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 7776156bd460..baf907a6f914 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -61,6 +61,9 @@ def test_create_application_client(self): pipeline_options = PipelineOptions() apiclient.DataflowApplicationClient(pipeline_options) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_pipeline_url(self): pipeline_options = PipelineOptions([ '--subnetwork', @@ -92,6 +95,9 @@ def test_pipeline_url(self): self.assertEqual(pipeline_url.string_value, FAKE_PIPELINE_URL) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_set_network(self): pipeline_options = PipelineOptions([ '--network', @@ -105,6 +111,9 @@ def test_set_network(self): FAKE_PIPELINE_URL) self.assertEqual(env.proto.workerPools[0].network, 'anetworkname') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_set_subnetwork(self): pipeline_options = PipelineOptions([ '--subnetwork', @@ -121,6 +130,9 @@ def test_set_subnetwork(self): env.proto.workerPools[0].subnetwork, '/regions/MY/subnetworks/SUBNETWORK') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_flexrs_blank(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp']) @@ -131,6 +143,9 @@ def test_flexrs_blank(self): FAKE_PIPELINE_URL) self.assertEqual(env.proto.flexResourceSchedulingGoal, None) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_flexrs_cost(self): pipeline_options = PipelineOptions([ '--flexrs_goal', @@ -149,6 +164,9 @@ def test_flexrs_cost(self): dataflow.Environment.FlexResourceSchedulingGoalValueValuesEnum. FLEXRS_COST_OPTIMIZED)) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_flexrs_speed(self): pipeline_options = PipelineOptions([ '--flexrs_goal', @@ -167,6 +185,9 @@ def test_flexrs_speed(self): dataflow.Environment.FlexResourceSchedulingGoalValueValuesEnum. FLEXRS_SPEED_OPTIMIZED)) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_sdk_harness_container_images_get_set(self): pipeline_options = PipelineOptions([ @@ -350,6 +371,9 @@ def test_translate_means_using_distribution_accumulator(self): self.assertEqual( metric_update.floatingPointMean.count.lowBits, accumulator.count) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_default_ip_configuration(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp']) @@ -359,6 +383,9 @@ def test_default_ip_configuration(self): FAKE_PIPELINE_URL) self.assertEqual(env.proto.workerPools[0].ipConfiguration, None) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_public_ip_configuration(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp', '--use_public_ips']) @@ -370,6 +397,9 @@ def test_public_ip_configuration(self): env.proto.workerPools[0].ipConfiguration, dataflow.WorkerPool.IpConfigurationValueValuesEnum.WORKER_IP_PUBLIC) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_private_ip_configuration(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp', '--no_use_public_ips']) @@ -381,6 +411,9 @@ def test_private_ip_configuration(self): env.proto.workerPools[0].ipConfiguration, dataflow.WorkerPool.IpConfigurationValueValuesEnum.WORKER_IP_PRIVATE) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_number_of_worker_harness_threads(self): pipeline_options = PipelineOptions([ '--temp_location', @@ -398,6 +431,9 @@ def test_number_of_worker_harness_threads(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_default_in_released_sdks(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp', '--streaming']) @@ -416,6 +452,9 @@ def test_harness_override_default_in_released_sdks(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_absent_in_released_sdks_with_runner_v2(self): pipeline_options = PipelineOptions([ '--temp_location', @@ -436,6 +475,9 @@ def test_harness_override_absent_in_released_sdks_with_runner_v2(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_custom_in_released_sdks(self): pipeline_options = PipelineOptions([ '--temp_location', @@ -460,6 +502,9 @@ def test_harness_override_custom_in_released_sdks(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_custom_in_released_sdks_with_runner_v2(self): pipeline_options = PipelineOptions([ '--temp_location', @@ -485,6 +530,9 @@ def test_harness_override_custom_in_released_sdks_with_runner_v2(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0.rc1') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_uses_base_version_in_rc_releases(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp', '--streaming']) @@ -503,6 +551,9 @@ def test_harness_override_uses_base_version_in_rc_releases(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0.dev') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_harness_override_absent_in_unreleased_sdk(self): pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp', '--streaming']) @@ -518,6 +569,9 @@ def test_harness_override_absent_in_unreleased_sdk(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0.dev') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_pinned_worker_harness_image_tag_used_in_dev_sdk(self): # streaming, fnapi pipeline. pipeline_options = PipelineOptions( @@ -580,6 +634,9 @@ def test_pinned_worker_harness_image_tag_used_in_dev_sdk(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_worker_harness_image_tag_matches_released_sdk_version(self): # streaming, fnapi pipeline. pipeline_options = PipelineOptions( @@ -630,6 +687,9 @@ def test_worker_harness_image_tag_matches_released_sdk_version(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0.rc1') + @unittest.skipIf( + sys.version_info.minor == 8, 'Re-enable once BEAM-9754 is ' + 'resolved') def test_worker_harness_image_tag_matches_base_sdk_version_of_an_rc(self): # streaming, fnapi pipeline. pipeline_options = PipelineOptions( @@ -676,6 +736,9 @@ def test_worker_harness_image_tag_matches_base_sdk_version_of_an_rc(self): names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/python%d%d:2.2.0' % (sys.version_info[0], sys.version_info[1]))) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_worker_harness_override_takes_precedence_over_sdk_defaults(self): # streaming, fnapi pipeline. pipeline_options = PipelineOptions([ @@ -765,6 +828,9 @@ def test_labels(self): self.assertEqual('key5', job.proto.labels.additionalProperties[4].key) self.assertEqual('', job.proto.labels.additionalProperties[4].value) + @unittest.skipIf( + sys.version_info.minor == 8, + 'Doesn\'t work on Python 3.8, see: BEAM-9754') def test_experiment_use_multiple_sdk_containers(self): pipeline_options = PipelineOptions([ '--project', @@ -944,7 +1010,8 @@ def test_stage_resources(self): '--temp_location', 'gs://test-location/temp', '--staging_location', - 'gs://test-location/staging' + 'gs://test-location/staging', + '--no_auth' ]) pipeline = beam_runner_api_pb2.Pipeline( components=beam_runner_api_pb2.Components( diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index 9be60ff497fa..446f00cda49b 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -40,10 +40,10 @@ # Update this version to the next version whenever there is a change that will # require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20200518' +BEAM_CONTAINER_VERSION = 'beam-master-20200521' # Update this version to the next version whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20200518' +BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20200521' # TODO(BEAM-5939): Remove these shared names once Dataflow worker is updated. PICKLED_MAIN_SESSION_FILE = 'pickled_main_session' diff --git a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py index 9d6e5d35467f..75b1db0b6341 100644 --- a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py +++ b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py @@ -21,6 +21,8 @@ from __future__ import absolute_import +from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import StandardOptions from apache_beam.pipeline import PTransformOverride @@ -184,3 +186,137 @@ def expand(self, pbegin): # will choose the incorrect coder for this transform. return Read(ptransform.source).with_output_types( ptransform.source.coder.to_type_hint()) + + +class WriteToBigQueryPTransformOverride(PTransformOverride): + def __init__(self, pipeline, options): + super(WriteToBigQueryPTransformOverride, self).__init__() + self.options = options + self.outputs = [] + + self._check_bq_outputs(pipeline) + + def _check_bq_outputs(self, pipeline): + """Checks that there are no consumers if the transform will be replaced. + + The WriteToBigQuery replacement is the native BigQuerySink which has an + output of a PDone. The original transform, however, returns a dict. The user + may be inadvertantly using the dict output which will have no side-effects + or fail pipeline construction esoterically. This checks the outputs and + gives a user-friendsly error. + """ + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.pipeline import PipelineVisitor + from apache_beam.io import WriteToBigQuery + + # First, retrieve all the outpts from all the WriteToBigQuery transforms + # that will be replaced. Later, we will use these to make sure no one + # consumes these. + class GetWriteToBqOutputsVisitor(PipelineVisitor): + def __init__(self, matches): + self.matches = matches + self.outputs = set() + + def enter_composite_transform(self, transform_node): + # Only add outputs that are going to be replaced. + if self.matches(transform_node): + self.outputs.update(set(transform_node.outputs.values())) + + outputs_visitor = GetWriteToBqOutputsVisitor(self.matches) + pipeline.visit(outputs_visitor) + + # Finally, verify that there are no consumers to the previously found + # outputs. + class VerifyWriteToBqOutputsVisitor(PipelineVisitor): + def __init__(self, outputs): + self.outputs = outputs + + def enter_composite_transform(self, transform_node): + self.visit_transform(transform_node) + + def visit_transform(self, transform_node): + if [o for o in self.outputs if o in transform_node.inputs]: + raise ValueError( + 'WriteToBigQuery was being replaced with the native ' + 'BigQuerySink, but the transform "{}" has an input which will be ' + 'replaced with a PDone. To fix, please remove all transforms ' + 'that read from any WriteToBigQuery transforms.'.format( + transform_node.full_label)) + + pipeline.visit(VerifyWriteToBqOutputsVisitor(outputs_visitor.outputs)) + + def matches(self, applied_ptransform): + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam import io + from apache_beam.runners.dataflow.internal import apiclient + + transform = applied_ptransform.transform + if (not isinstance(transform, io.WriteToBigQuery) or + getattr(transform, 'override', False)): + return False + + use_fnapi = apiclient._use_fnapi(self.options) + experiments = self.options.view_as(DebugOptions).experiments or [] + if (use_fnapi or 'use_beam_bq_sink' in experiments): + return False + + if transform.schema == io.gcp.bigquery.SCHEMA_AUTODETECT: + raise RuntimeError( + 'Schema auto-detection is not supported on the native sink') + + # The replacement is only valid for Batch. + standard_options = self.options.view_as(StandardOptions) + if standard_options.streaming: + if transform.write_disposition == io.BigQueryDisposition.WRITE_TRUNCATE: + raise RuntimeError('Can not use write truncation mode in streaming') + return False + + self.outputs = list(applied_ptransform.outputs.keys()) + return True + + def get_replacement_transform(self, ptransform): + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam import io + + class WriteToBigQuery(io.WriteToBigQuery): + override = True + + def __init__(self, transform, outputs): + self.transform = transform + self.outputs = outputs + + def __getattr__(self, name): + """Returns the given attribute from the parent. + + This allows this transform to act like a WriteToBigQuery transform + without having to construct a new WriteToBigQuery transform. + """ + return self.transform.__getattribute__(name) + + def expand(self, pcoll): + from apache_beam.io.gcp.bigquery_tools import parse_table_schema_from_json + import json + + schema = None + if self.schema: + schema = parse_table_schema_from_json(json.dumps(self.schema)) + + out = pcoll | io.Write( + io.BigQuerySink( + self.table_reference.tableId, + self.table_reference.datasetId, + self.table_reference.projectId, + schema, + self.create_disposition, + self.write_disposition, + kms_key=self.kms_key)) + + # The WriteToBigQuery can have different outputs depending on if it's + # Batch or Streaming. This retrieved the output keys from the node and + # is replacing them here to be consistent. + return {key: out for key in self.outputs} + + return WriteToBigQuery(ptransform, self.outputs) diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py index 54e39a5ee21a..3c173924f006 100644 --- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py @@ -22,6 +22,7 @@ from __future__ import absolute_import import json +import sys import tempfile import unittest @@ -40,6 +41,8 @@ @unittest.skipIf(apiclient is None, 'GCP dependencies are not installed') +@unittest.skipIf( + sys.version_info.minor == 8, 'Doesn\'t work on Python 3.8, see: BEAM-9754') class TemplatingDataflowRunnerTest(unittest.TestCase): """TemplatingDataflow tests.""" def test_full_completion(self): diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index 5181ad734ff0..51b064a88114 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -50,10 +50,8 @@ from apache_beam.transforms.core import CombineValuesDoFn from apache_beam.transforms.core import DoFn from apache_beam.transforms.core import ParDo -from apache_beam.transforms.core import _GroupAlsoByWindow -from apache_beam.transforms.core import _GroupAlsoByWindowDoFn -from apache_beam.transforms.core import _GroupByKeyOnly from apache_beam.transforms.ptransform import PTransform +from apache_beam.typehints import trivial_inference # Note that the BundleBasedDirectRunner and SwitchingDirectRunner names are # experimental and have no backwards compatibility guarantees. @@ -135,6 +133,57 @@ def visit_transform(self, applied_ptransform): V = typing.TypeVar('V') +@typehints.with_input_types(typing.Tuple[K, V]) +@typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) +class _GroupByKeyOnly(PTransform): + """A group by key transform, ignoring windows.""" + def infer_output_type(self, input_type): + key_type, value_type = trivial_inference.key_value_types(input_type) + return typehints.KV[key_type, typehints.Iterable[value_type]] + + def expand(self, pcoll): + self._check_pcollection(pcoll) + return PCollection.from_(pcoll) + + +@typehints.with_input_types(typing.Tuple[K, typing.Iterable[V]]) +@typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) +class _GroupAlsoByWindow(ParDo): + """The GroupAlsoByWindow transform.""" + def __init__(self, windowing): + super(_GroupAlsoByWindow, self).__init__(_GroupAlsoByWindowDoFn(windowing)) + self.windowing = windowing + + def expand(self, pcoll): + self._check_pcollection(pcoll) + return PCollection.from_(pcoll) + + +class _GroupAlsoByWindowDoFn(DoFn): + # TODO(robertwb): Support combiner lifting. + + def __init__(self, windowing): + super(_GroupAlsoByWindowDoFn, self).__init__() + self.windowing = windowing + + def infer_output_type(self, input_type): + key_type, windowed_value_iter_type = trivial_inference.key_value_types( + input_type) + value_type = windowed_value_iter_type.inner_type.inner_type + return typehints.Iterable[typehints.KV[key_type, + typehints.Iterable[value_type]]] + + def start_bundle(self): + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.transforms.trigger import create_trigger_driver + # pylint: enable=wrong-import-order, wrong-import-position + self.driver = create_trigger_driver(self.windowing, True) + + def process(self, element): + k, vs = element + return self.driver.process_entire_key(k, vs) + + @typehints.with_input_types(typing.Tuple[K, V]) @typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) class _StreamingGroupByKeyOnly(_GroupByKeyOnly): @@ -172,6 +221,59 @@ def from_runner_api_parameter(unused_ptransform, payload, context): context.windowing_strategies.get_by_id(payload.value)) +@typehints.with_input_types(typing.Tuple[K, typing.Iterable[V]]) +@typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) +class _GroupByKey(PTransform): + """The DirectRunner GroupByKey implementation.""" + def expand(self, pcoll): + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.coders import typecoders + + input_type = pcoll.element_type + if input_type is not None: + # Initialize type-hints used below to enforce type-checking and to + # pass downstream to further PTransforms. + key_type, value_type = trivial_inference.key_value_types(input_type) + # Enforce the input to a GBK has a KV element type. + pcoll.element_type = typehints.typehints.coerce_to_kv_type( + pcoll.element_type) + typecoders.registry.verify_deterministic( + typecoders.registry.get_coder(key_type), + 'GroupByKey operation "%s"' % self.label) + + reify_output_type = typehints.KV[ + key_type, typehints.WindowedValue[value_type]] # type: ignore[misc] + gbk_input_type = ( + typehints. + KV[key_type, + typehints.Iterable[ + typehints.WindowedValue[ # type: ignore[misc] + value_type]]]) + gbk_output_type = typehints.KV[key_type, typehints.Iterable[value_type]] + + # pylint: disable=bad-continuation + return ( + pcoll + | 'ReifyWindows' >> ( + ParDo(beam.GroupByKey.ReifyWindows()).with_output_types( + reify_output_type)) + | 'GroupByKey' >> ( + _GroupByKeyOnly().with_input_types( + reify_output_type).with_output_types(gbk_input_type)) + | ( + 'GroupByWindow' >> + _GroupAlsoByWindow(pcoll.windowing).with_input_types( + gbk_input_type).with_output_types(gbk_output_type))) + else: + # The input_type is None, run the default + return ( + pcoll + | 'ReifyWindows' >> ParDo(beam.GroupByKey.ReifyWindows()) + | 'GroupByKey' >> _GroupByKeyOnly() + | 'GroupByWindow' >> _GroupAlsoByWindow(pcoll.windowing)) + + def _get_transform_overrides(pipeline_options): # A list of PTransformOverride objects to be applied before running a pipeline # using DirectRunner. @@ -234,7 +336,25 @@ def get_replacement_transform(self, transform): from apache_beam.runners.direct.test_stream_impl import _ExpandableTestStream return _ExpandableTestStream(transform) + class GroupByKeyPTransformOverride(PTransformOverride): + """A ``PTransformOverride`` for ``GroupByKey``. + + This replaces the Beam implementation as a primitive. + """ + def matches(self, applied_ptransform): + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.transforms.core import GroupByKey + return isinstance(applied_ptransform.transform, GroupByKey) + + def get_replacement_transform(self, ptransform): + return _GroupByKey() + overrides = [ + # This needs to be the first and the last override. Other overrides depend + # on the GroupByKey implementation to be composed of _GroupByKeyOnly and + # _GroupAlsoByWindow. + GroupByKeyPTransformOverride(), SplittableParDoOverride(), ProcessKeyedElementsViaKeyedWorkItemsOverride(), CombinePerKeyOverride(), @@ -253,6 +373,10 @@ def get_replacement_transform(self, transform): except ImportError: pass + # This also needs to be last because other transforms apply GBKs which need to + # be translated into a DirectRunner-compatible transform. + overrides.append(GroupByKeyPTransformOverride()) + return overrides diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index bc6f397968ba..6421b3935e6c 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -45,6 +45,7 @@ from apache_beam.runners.common import DoFnState from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access from apache_beam.runners.direct.direct_runner import _DirectReadFromPubSub +from apache_beam.runners.direct.direct_runner import _GroupByKeyOnly from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly from apache_beam.runners.direct.direct_userstate import DirectUserStateContext @@ -106,7 +107,7 @@ def __init__(self, evaluation_context): core.Flatten: _FlattenEvaluator, core.Impulse: _ImpulseEvaluator, core.ParDo: _ParDoEvaluator, - core._GroupByKeyOnly: _GroupByKeyOnlyEvaluator, + _GroupByKeyOnly: _GroupByKeyOnlyEvaluator, _StreamingGroupByKeyOnly: _StreamingGroupByKeyOnlyEvaluator, _StreamingGroupAlsoByWindow: _StreamingGroupAlsoByWindowEvaluator, _NativeWrite: _NativeWriteEvaluator, @@ -176,7 +177,7 @@ def should_execute_serially(self, applied_ptransform): True if executor should execute applied_ptransform serially. """ if isinstance(applied_ptransform.transform, - (core._GroupByKeyOnly, + (_GroupByKeyOnly, _StreamingGroupByKeyOnly, _StreamingGroupAlsoByWindow, _NativeWrite)): diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py index fdc9145aef10..5cf0921ac804 100644 --- a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py +++ b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py @@ -219,7 +219,7 @@ def test_read_cache_expansion(self): pipeline_proto = to_stable_runner_api(p) pipeline_info = pipeline_analyzer.PipelineInfo(pipeline_proto.components) - pcoll_id = 'ref_PCollection_PCollection_12' # Output PCollection of Square + pcoll_id = 'ref_PCollection_PCollection_10' # Output PCollection of Square cache_label1 = pipeline_info.cache_label(pcoll_id) analyzer = pipeline_analyzer.PipelineAnalyzer( diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py b/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py index 8ef6f8c65cf7..9f2684e8e83f 100644 --- a/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py +++ b/sdks/python/apache_beam/runners/interactive/pipeline_instrument_test.py @@ -69,7 +69,7 @@ def test_cacheable_key_without_version_map(self): _, ctx = p.to_runner_api(use_fake_coders=True, return_context=True) self.assertEqual( instr.cacheable_key(init_pcoll, instr.pcolls_to_pcoll_id(p, ctx)), - str(id(init_pcoll)) + '_ref_PCollection_PCollection_10') + str(id(init_pcoll)) + '_ref_PCollection_PCollection_8') def test_cacheable_key_with_version_map(self): p = beam.Pipeline(interactive_runner.InteractiveRunner()) @@ -94,8 +94,8 @@ def test_cacheable_key_with_version_map(self): instr.cacheable_key( init_pcoll_2, instr.pcolls_to_pcoll_id(p2, ctx), - {'ref_PCollection_PCollection_10': str(id(init_pcoll))}), - str(id(init_pcoll)) + '_ref_PCollection_PCollection_10') + {'ref_PCollection_PCollection_8': str(id(init_pcoll))}), + str(id(init_pcoll)) + '_ref_PCollection_PCollection_8') def test_cache_key(self): p = beam.Pipeline(interactive_runner.InteractiveRunner()) @@ -131,19 +131,19 @@ def test_cacheables(self): pipeline_instrument._cacheable_key(init_pcoll): instr.Cacheable( var='init_pcoll', version=str(id(init_pcoll)), - pcoll_id='ref_PCollection_PCollection_10', + pcoll_id='ref_PCollection_PCollection_8', producer_version=str(id(init_pcoll.producer)), pcoll=init_pcoll), pipeline_instrument._cacheable_key(squares): instr.Cacheable( var='squares', version=str(id(squares)), - pcoll_id='ref_PCollection_PCollection_11', + pcoll_id='ref_PCollection_PCollection_9', producer_version=str(id(squares.producer)), pcoll=squares), pipeline_instrument._cacheable_key(cubes): instr.Cacheable( var='cubes', version=str(id(cubes)), - pcoll_id='ref_PCollection_PCollection_12', + pcoll_id='ref_PCollection_PCollection_10', producer_version=str(id(cubes.producer)), pcoll=cubes) }) diff --git a/sdks/python/apache_beam/runners/portability/artifact_service_test.py b/sdks/python/apache_beam/runners/portability/artifact_service_test.py index 14993dcaabb1..6a6768ad6e0f 100644 --- a/sdks/python/apache_beam/runners/portability/artifact_service_test.py +++ b/sdks/python/apache_beam/runners/portability/artifact_service_test.py @@ -44,7 +44,7 @@ from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners.portability import artifact_service from apache_beam.utils import proto_utils -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor class AbstractArtifactServiceTest(unittest.TestCase): @@ -83,7 +83,7 @@ def test_basic(self): self._run_staging(self._service, self._service) def test_with_grpc(self): - server = grpc.server(UnboundedThreadPoolExecutor()) + server = grpc.server(thread_pool_executor.shared_unbounded_instance()) try: beam_artifact_api_pb2_grpc.add_LegacyArtifactStagingServiceServicer_to_server( self._service, server) @@ -225,7 +225,7 @@ def check(index): self._service, tokens[session(index)], name(index))) # pylint: disable=range-builtin-not-iterating - pool = UnboundedThreadPoolExecutor() + pool = thread_pool_executor.shared_unbounded_instance() sessions = set(pool.map(put, range(100))) tokens = dict(pool.map(commit, sessions)) # List forces materialization. diff --git a/sdks/python/apache_beam/runners/portability/expansion_service_test.py b/sdks/python/apache_beam/runners/portability/expansion_service_test.py index 6a7ab579c54d..a1b531db13be 100644 --- a/sdks/python/apache_beam/runners/portability/expansion_service_test.py +++ b/sdks/python/apache_beam/runners/portability/expansion_service_test.py @@ -36,7 +36,7 @@ from apache_beam.runners.portability import expansion_service from apache_beam.transforms import ptransform from apache_beam.transforms.external import ImplicitSchemaPayloadBuilder -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor # This script provides an expansion service and example ptransforms for running # external transform test cases. See external_test.py for details. @@ -296,7 +296,7 @@ def main(unused_argv): '-p', '--port', type=int, help='port on which to serve the job api') options = parser.parse_args() global server - server = grpc.server(UnboundedThreadPoolExecutor()) + server = grpc.server(thread_pool_executor.shared_unbounded_instance()) beam_expansion_api_pb2_grpc.add_ExpansionServiceServicer_to_server( expansion_service.ExpansionServiceServicer( PipelineOptions( diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py index 58db57c9157f..c2147ce44182 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py @@ -64,7 +64,7 @@ from apache_beam.transforms import environments from apache_beam.utils import profiler from apache_beam.utils import proto_utils -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor if TYPE_CHECKING: from apache_beam.pipeline import Pipeline @@ -925,7 +925,7 @@ def execute(part_map_input_timers): expected_output_timers, dry_run) - with UnboundedThreadPoolExecutor() as executor: + with thread_pool_executor.shared_unbounded_instance() as executor: for result, split_result in executor.map(execute, zip(part_inputs, # pylint: disable=zip-builtin-not-iterating timer_inputs)): split_result_list += split_result diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py index 221d28538dfe..a49ce03e42c9 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py @@ -1202,7 +1202,7 @@ def has_mi_for_ptransform(mon_infos, ptransform): # postgbk monitoring infos labels = { - monitoring_infos.PCOLLECTION_LABEL: 'ref_PCollection_PCollection_8' + monitoring_infos.PCOLLECTION_LABEL: 'ref_PCollection_PCollection_6' } self.assert_has_counter( postgbk_mis, monitoring_infos.ELEMENT_COUNT_URN, labels, value=1) @@ -1210,7 +1210,7 @@ def has_mi_for_ptransform(mon_infos, ptransform): postgbk_mis, monitoring_infos.SAMPLED_BYTE_SIZE_URN, labels) labels = { - monitoring_infos.PCOLLECTION_LABEL: 'ref_PCollection_PCollection_9' + monitoring_infos.PCOLLECTION_LABEL: 'ref_PCollection_PCollection_7' } self.assert_has_counter( postgbk_mis, monitoring_infos.ELEMENT_COUNT_URN, labels, value=5) diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py index 6808b47823b4..6a276565f70b 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py @@ -63,7 +63,7 @@ from apache_beam.runners.worker.sdk_worker import _Future from apache_beam.runners.worker.statecache import StateCache from apache_beam.utils import proto_utils -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor # State caching is enabled in the fn_api_runner for testing, except for one # test which runs without state caching (FnApiRunnerTestWithDisabledCaching). @@ -441,7 +441,8 @@ def __init__(self, # type: (...) -> None self.state = state self.provision_info = provision_info - self.control_server = grpc.server(UnboundedThreadPoolExecutor()) + self.control_server = grpc.server( + thread_pool_executor.shared_unbounded_instance()) self.control_port = self.control_server.add_insecure_port('[::]:0') self.control_address = 'localhost:%s' % self.control_port @@ -451,11 +452,13 @@ def __init__(self, no_max_message_sizes = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] self.data_server = grpc.server( - UnboundedThreadPoolExecutor(), options=no_max_message_sizes) + thread_pool_executor.shared_unbounded_instance(), + options=no_max_message_sizes) self.data_port = self.data_server.add_insecure_port('[::]:0') self.state_server = grpc.server( - UnboundedThreadPoolExecutor(), options=no_max_message_sizes) + thread_pool_executor.shared_unbounded_instance(), + options=no_max_message_sizes) self.state_port = self.state_server.add_insecure_port('[::]:0') self.control_handler = BeamFnControlServicer(worker_manager) @@ -493,7 +496,8 @@ def __init__(self, GrpcStateServicer(state), self.state_server) self.logging_server = grpc.server( - UnboundedThreadPoolExecutor(), options=no_max_message_sizes) + thread_pool_executor.shared_unbounded_instance(), + options=no_max_message_sizes) self.logging_port = self.logging_server.add_insecure_port('[::]:0') beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( BasicLoggingService(), self.logging_server) diff --git a/sdks/python/apache_beam/runners/portability/local_job_service.py b/sdks/python/apache_beam/runners/portability/local_job_service.py index b99bfeb183ab..dcdc4425fcba 100644 --- a/sdks/python/apache_beam/runners/portability/local_job_service.py +++ b/sdks/python/apache_beam/runners/portability/local_job_service.py @@ -48,7 +48,7 @@ from apache_beam.runners.portability import artifact_service from apache_beam.runners.portability.fn_api_runner import fn_runner from apache_beam.runners.portability.fn_api_runner import worker_handlers -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor if TYPE_CHECKING: from google.protobuf import struct_pb2 # pylint: disable=ungrouped-imports @@ -141,7 +141,7 @@ def get_service_address(self): return 'localhost' def start_grpc_server(self, port=0): - self._server = grpc.server(UnboundedThreadPoolExecutor()) + self._server = grpc.server(thread_pool_executor.shared_unbounded_instance()) port = self._server.add_insecure_port( '%s:%d' % (self.get_bind_address(), port)) beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(self, self._server) @@ -194,7 +194,8 @@ def __init__( self._worker_id = worker_id def run(self): - logging_server = grpc.server(UnboundedThreadPoolExecutor()) + logging_server = grpc.server( + thread_pool_executor.shared_unbounded_instance()) logging_port = logging_server.add_insecure_port('[::]:0') logging_server.start() logging_servicer = BeamFnLoggingServicer() diff --git a/sdks/python/apache_beam/runners/portability/portable_stager_test.py b/sdks/python/apache_beam/runners/portability/portable_stager_test.py index c7a8e1cfc674..44fb4986649e 100644 --- a/sdks/python/apache_beam/runners/portability/portable_stager_test.py +++ b/sdks/python/apache_beam/runners/portability/portable_stager_test.py @@ -36,7 +36,7 @@ from apache_beam.portability.api import beam_artifact_api_pb2 from apache_beam.portability.api import beam_artifact_api_pb2_grpc from apache_beam.runners.portability import portable_stager -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor class PortableStagerTest(unittest.TestCase): @@ -58,7 +58,7 @@ def _stage_files(self, files): describing the name of the artifacts in local temp folder and desired name in staging location. """ - server = grpc.server(UnboundedThreadPoolExecutor()) + server = grpc.server(thread_pool_executor.shared_unbounded_instance()) staging_service = TestLocalFileSystemLegacyArtifactStagingServiceServicer( self._remote_dir) beam_artifact_api_pb2_grpc.add_LegacyArtifactStagingServiceServicer_to_server( diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index a4922a55a3ca..4f09df350ba9 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -34,7 +34,7 @@ from apache_beam.runners.worker import data_plane from apache_beam.runners.worker.worker_id_interceptor import WorkerIdInterceptor from apache_beam.testing.util import timeout -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor class DataChannelTest(unittest.TestCase): @@ -56,7 +56,7 @@ def _grpc_data_channel_test(self, time_based_flush=False): data_channel_service = \ data_servicer.get_conn_by_worker_id(worker_id) - server = grpc.server(UnboundedThreadPoolExecutor()) + server = grpc.server(thread_pool_executor.shared_unbounded_instance()) beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server(data_servicer, server) test_port = server.add_insecure_port('[::]:0') server.start() diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 4d9291d99e68..80fe543e3335 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -32,7 +32,7 @@ from apache_beam.runners.common import NameContext from apache_beam.runners.worker import log_handler from apache_beam.runners.worker import statesampler -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor _LOGGER = logging.getLogger(__name__) @@ -52,7 +52,7 @@ def Logging(self, request_iterator, context): class FnApiLogRecordHandlerTest(unittest.TestCase): def setUp(self): self.test_logging_service = BeamFnLoggingServicer() - self.server = grpc.server(UnboundedThreadPoolExecutor()) + self.server = grpc.server(thread_pool_executor.shared_unbounded_instance()) beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server( self.test_logging_service, self.server) self.test_port = self.server.add_insecure_port('[::]:0') diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index eed471d0bc7e..a58dd2facf21 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -64,7 +64,7 @@ from apache_beam.runners.worker.statecache import StateCache from apache_beam.runners.worker.worker_id_interceptor import WorkerIdInterceptor from apache_beam.runners.worker.worker_status import FnApiWorkerStatusHandler -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor if TYPE_CHECKING: from apache_beam.portability.api import endpoints_pb2 @@ -186,10 +186,11 @@ def __init__(self, else: self._status_handler = None - # TODO(BEAM-8998) use common UnboundedThreadPoolExecutor to process bundle - # progress once dataflow runner's excessive progress polling is removed. + # TODO(BEAM-8998) use common + # thread_pool_executor.shared_unbounded_instance() to process bundle + # progress once dataflow runner's excessive progress polling is removed. self._report_progress_executor = futures.ThreadPoolExecutor(max_workers=1) - self._worker_thread_pool = UnboundedThreadPoolExecutor() + self._worker_thread_pool = thread_pool_executor.shared_unbounded_instance() self._responses = queue.Queue( ) # type: queue.Queue[beam_fn_api_pb2.InstructionResponse] _LOGGER.info('Initializing SDKHarness with unbounded number of workers.') diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index a7c3bc394674..d9b808354f20 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -38,7 +38,7 @@ from apache_beam.portability.api import metrics_pb2 from apache_beam.runners.worker import sdk_worker from apache_beam.runners.worker import statecache -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor _LOGGER = logging.getLogger(__name__) @@ -102,7 +102,7 @@ def _check_fn_registration_multi_request(self, *args): test_controller = BeamFnControlServicer(requests) - server = grpc.server(UnboundedThreadPoolExecutor()) + server = grpc.server(thread_pool_executor.shared_unbounded_instance()) beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server( test_controller, server) test_port = server.add_insecure_port("[::]:0") diff --git a/sdks/python/apache_beam/runners/worker/worker_pool_main.py b/sdks/python/apache_beam/runners/worker/worker_pool_main.py index b18961a391bd..f4a4728c373d 100644 --- a/sdks/python/apache_beam/runners/worker/worker_pool_main.py +++ b/sdks/python/apache_beam/runners/worker/worker_pool_main.py @@ -48,7 +48,7 @@ from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.portability.api import beam_fn_api_pb2_grpc from apache_beam.runners.worker import sdk_worker -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor _LOGGER = logging.getLogger(__name__) @@ -78,7 +78,8 @@ def start( container_executable=None # type: Optional[str] ): # type: (...) -> Tuple[str, grpc.Server] - worker_server = grpc.server(UnboundedThreadPoolExecutor()) + worker_server = grpc.server( + thread_pool_executor.shared_unbounded_instance()) worker_address = 'localhost:%s' % worker_server.add_insecure_port( '[::]:%s' % port) worker_pool = cls( diff --git a/sdks/python/apache_beam/runners/worker/worker_status_test.py b/sdks/python/apache_beam/runners/worker/worker_status_test.py index 0872de4787b0..fea0d7371288 100644 --- a/sdks/python/apache_beam/runners/worker/worker_status_test.py +++ b/sdks/python/apache_beam/runners/worker/worker_status_test.py @@ -28,7 +28,7 @@ from apache_beam.portability.api import beam_fn_api_pb2_grpc from apache_beam.runners.worker.worker_status import FnApiWorkerStatusHandler from apache_beam.testing.util import timeout -from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor +from apache_beam.utils import thread_pool_executor class BeamFnStatusServicer(beam_fn_api_pb2_grpc.BeamFnWorkerStatusServicer): @@ -52,7 +52,7 @@ class FnApiWorkerStatusHandlerTest(unittest.TestCase): def setUp(self): self.num_request = 3 self.test_status_service = BeamFnStatusServicer(self.num_request) - self.server = grpc.server(UnboundedThreadPoolExecutor()) + self.server = grpc.server(thread_pool_executor.shared_unbounded_instance()) beam_fn_api_pb2_grpc.add_BeamFnWorkerStatusServicer_to_server( self.test_status_service, self.server) self.test_port = self.server.add_insecure_port('[::]:0') diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 92b88f95e244..fe87117d1e43 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -2226,50 +2226,11 @@ def infer_output_type(self, input_type): key_type, typehints.WindowedValue[value_type]]] # type: ignore[misc] def expand(self, pcoll): - # This code path is only used in the local direct runner. For Dataflow - # runner execution, the GroupByKey transform is expanded on the service. - input_type = pcoll.element_type - if input_type is not None: - # Initialize type-hints used below to enforce type-checking and to pass - # downstream to further PTransforms. - key_type, value_type = trivial_inference.key_value_types(input_type) - # Enforce the input to a GBK has a KV element type. - pcoll.element_type = typehints.KV[key_type, value_type] - typecoders.registry.verify_deterministic( - typecoders.registry.get_coder(key_type), - 'GroupByKey operation "%s"' % self.label) - - reify_output_type = typehints.KV[ - key_type, typehints.WindowedValue[value_type]] # type: ignore[misc] - gbk_input_type = ( - typehints.KV[ - key_type, - typehints.Iterable[typehints.WindowedValue[ # type: ignore[misc] - value_type]]]) - gbk_output_type = typehints.KV[key_type, typehints.Iterable[value_type]] - - # pylint: disable=bad-continuation - return ( - pcoll - | 'ReifyWindows' >> - (ParDo(self.ReifyWindows()).with_output_types(reify_output_type)) - | 'GroupByKey' >> ( - _GroupByKeyOnly().with_input_types( - reify_output_type).with_output_types(gbk_input_type)) - | ( - 'GroupByWindow' >> - _GroupAlsoByWindow(pcoll.windowing).with_input_types( - gbk_input_type).with_output_types(gbk_output_type))) - else: - # The input_type is None, run the default - return ( - pcoll - | 'ReifyWindows' >> ParDo(self.ReifyWindows()) - | 'GroupByKey' >> _GroupByKeyOnly() - | 'GroupByWindow' >> _GroupAlsoByWindow(pcoll.windowing)) + return pvalue.PCollection.from_(pcoll) def infer_output_type(self, input_type): - key_type, value_type = trivial_inference.key_value_types(input_type) + key_type, value_type = (typehints.typehints.coerce_to_kv_type( + input_type).tuple_types) return typehints.KV[key_type, typehints.Iterable[value_type]] def to_runner_api_parameter(self, unused_context): @@ -2286,57 +2247,6 @@ def runner_api_requires_keyed_input(self): return True -@typehints.with_input_types(typing.Tuple[K, V]) -@typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) -class _GroupByKeyOnly(PTransform): - """A group by key transform, ignoring windows.""" - def infer_output_type(self, input_type): - key_type, value_type = trivial_inference.key_value_types(input_type) - return typehints.KV[key_type, typehints.Iterable[value_type]] - - def expand(self, pcoll): - self._check_pcollection(pcoll) - return pvalue.PCollection.from_(pcoll) - - -@typehints.with_input_types(typing.Tuple[K, typing.Iterable[V]]) -@typehints.with_output_types(typing.Tuple[K, typing.Iterable[V]]) -class _GroupAlsoByWindow(ParDo): - """The GroupAlsoByWindow transform.""" - def __init__(self, windowing): - super(_GroupAlsoByWindow, self).__init__(_GroupAlsoByWindowDoFn(windowing)) - self.windowing = windowing - - def expand(self, pcoll): - self._check_pcollection(pcoll) - return pvalue.PCollection.from_(pcoll) - - -class _GroupAlsoByWindowDoFn(DoFn): - # TODO(robertwb): Support combiner lifting. - - def __init__(self, windowing): - super(_GroupAlsoByWindowDoFn, self).__init__() - self.windowing = windowing - - def infer_output_type(self, input_type): - key_type, windowed_value_iter_type = trivial_inference.key_value_types( - input_type) - value_type = windowed_value_iter_type.inner_type.inner_type - return typehints.Iterable[typehints.KV[key_type, - typehints.Iterable[value_type]]] - - def start_bundle(self): - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.transforms.trigger import create_trigger_driver - # pylint: enable=wrong-import-order, wrong-import-position - self.driver = create_trigger_driver(self.windowing, True) - - def process(self, element): - k, vs = element - return self.driver.process_entire_key(k, vs) - - class Partition(PTransformWithSideInputs): """Split a PCollection into several partitions. diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index ee007ff653f6..1cdd273fb3c4 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -52,7 +52,6 @@ from apache_beam.testing.util import equal_to from apache_beam.transforms import WindowInto from apache_beam.transforms import window -from apache_beam.transforms.core import _GroupByKeyOnly from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.ptransform import PTransform @@ -680,7 +679,7 @@ def test_group_by_key_only_input_must_be_kv_pairs(self): with self.assertRaises(typehints.TypeCheckError) as cm: with TestPipeline() as pipeline: pcolls = pipeline | 'A' >> beam.Create(['a', 'b', 'f']) - pcolls | 'D' >> _GroupByKeyOnly() + pcolls | 'D' >> beam.GroupByKey() expected_error_prefix = ( 'Input type hint violation at D: expected ' @@ -1233,7 +1232,7 @@ def test_group_by_key_only_output_type_deduction(self): | ( 'Pair' >> beam.Map(lambda x: (x, ord(x))).with_output_types( typing.Tuple[str, str])) - | _GroupByKeyOnly()) + | beam.GroupByKey()) # Output type should correctly be deduced. # GBK-only should deduce that Tuple[A, B] is turned into @@ -1261,7 +1260,7 @@ def test_group_by_key_only_does_not_type_check(self): ( self.p | beam.Create([1, 2, 3]).with_output_types(int) - | 'F' >> _GroupByKeyOnly()) + | 'F' >> beam.GroupByKey()) self.assertStartswith( e.exception.args[0], @@ -1309,7 +1308,7 @@ def test_pipeline_checking_gbk_insufficient_type_information(self): self.p | 'Nums' >> beam.Create(range(5)).with_output_types(int) | 'ModDup' >> beam.Map(lambda x: (x % 2, x)) - | _GroupByKeyOnly()) + | beam.GroupByKey()) self.assertEqual( 'Pipeline type checking is enabled, however no output ' @@ -2244,7 +2243,7 @@ def test_pardo_type_inference(self): def test_gbk_type_inference(self): self.assertEqual( typehints.Tuple[str, typehints.Iterable[int]], - _GroupByKeyOnly().infer_output_type(typehints.KV[str, int])) + beam.GroupByKey().infer_output_type(typehints.KV[str, int])) def test_pipeline_inference(self): created = self.p | beam.Create(['a', 'b', 'c']) diff --git a/sdks/python/apache_beam/transforms/sql.py b/sdks/python/apache_beam/transforms/sql.py index 86423761b8b6..ce409d21fd26 100644 --- a/sdks/python/apache_beam/transforms/sql.py +++ b/sdks/python/apache_beam/transforms/sql.py @@ -32,7 +32,8 @@ __all__ = ['SqlTransform'] SqlTransformSchema = typing.NamedTuple( - 'SqlTransformSchema', [('query', unicode)]) + 'SqlTransformSchema', [('query', unicode), + ('dialect', typing.Optional[unicode])]) class SqlTransform(ExternalTransform): @@ -66,9 +67,10 @@ class SqlTransform(ExternalTransform): """ URN = 'beam:external:java:sql:v1' - def __init__(self, query): + def __init__(self, query, dialect=None): super(SqlTransform, self).__init__( self.URN, - NamedTupleBasedPayloadBuilder(SqlTransformSchema(query=query)), + NamedTupleBasedPayloadBuilder( + SqlTransformSchema(query=query, dialect=dialect)), BeamJarExpansionService( ':sdks:java:extensions:sql:expansion-service:shadowJar')) diff --git a/sdks/python/apache_beam/transforms/sql_test.py b/sdks/python/apache_beam/transforms/sql_test.py index b8097145def0..918954cb84d7 100644 --- a/sdks/python/apache_beam/transforms/sql_test.py +++ b/sdks/python/apache_beam/transforms/sql_test.py @@ -37,10 +37,10 @@ from apache_beam.transforms.sql import SqlTransform SimpleRow = typing.NamedTuple( - "SimpleRow", [("int", int), ("str", unicode), ("flt", float)]) + "SimpleRow", [("id", int), ("str", unicode), ("flt", float)]) coders.registry.register_coder(SimpleRow, coders.RowCoder) -Enrich = typing.NamedTuple("Enrich", [("int", int), ("metadata", unicode)]) +Enrich = typing.NamedTuple("Enrich", [("id", int), ("metadata", unicode)]) coders.registry.register_coder(Enrich, coders.RowCoder) @@ -71,7 +71,7 @@ def test_generate_data(self): with TestPipeline() as p: out = p | SqlTransform( """SELECT - CAST(1 AS INT) AS `int`, + CAST(1 AS INT) AS `id`, CAST('foo' AS VARCHAR) AS `str`, CAST(3.14 AS DOUBLE) AS `flt`""") assert_that(out, equal_to([(1, "foo", 3.14)])) @@ -80,7 +80,7 @@ def test_project(self): with TestPipeline() as p: out = ( p | beam.Create([SimpleRow(1, "foo", 3.14)]) - | SqlTransform("SELECT `int`, `flt` FROM PCOLLECTION")) + | SqlTransform("SELECT `id`, `flt` FROM PCOLLECTION")) assert_that(out, equal_to([(1, 3.14)])) def test_filter(self): @@ -109,7 +109,7 @@ def test_agg(self): SELECT `str`, COUNT(*) AS `count`, - SUM(`int`) AS `sum`, + SUM(`id`) AS `sum`, AVG(`flt`) AS `avg` FROM PCOLLECTION GROUP BY `str`""")) assert_that(out, equal_to([("foo", 3, 3, 2), ("bar", 4, 8, 1.414)])) @@ -131,13 +131,23 @@ def test_tagged_join(self): | SqlTransform( """ SELECT - simple.`int` AS `int`, + simple.`id` AS `id`, enrich.metadata AS metadata FROM simple JOIN enrich - ON simple.`int` = enrich.`int`""")) + ON simple.`id` = enrich.`id`""")) assert_that(out, equal_to([(1, "a"), (26, "z"), (1, "a")])) + def test_zetasql_generate_data(self): + with TestPipeline() as p: + out = p | SqlTransform( + """SELECT + CAST(1 AS INT64) AS `int`, + CAST('foo' AS STRING) AS `str`, + CAST(3.14 AS FLOAT64) AS `flt`""", + dialect="zetasql") + assert_that(out, equal_to([(1, "foo", 3.14)])) + if __name__ == "__main__": logging.getLogger().setLevel(logging.INFO) diff --git a/sdks/python/apache_beam/typehints/schemas.py b/sdks/python/apache_beam/typehints/schemas.py index faff159ed17e..f69f4756deaa 100644 --- a/sdks/python/apache_beam/typehints/schemas.py +++ b/sdks/python/apache_beam/typehints/schemas.py @@ -115,12 +115,16 @@ def get_schema_by_id(self, unique_id): float: schema_pb2.DOUBLE, }) +# Name of the attribute added to user types (existing and generated) to store +# the corresponding schema ID +_BEAM_SCHEMA_ID = "_beam_schema_id" + def typing_to_runner_api(type_): if _match_is_named_tuple(type_): schema = None - if hasattr(type_, 'id'): - schema = SCHEMA_REGISTRY.get_schema_by_id(type_.id) + if hasattr(type_, _BEAM_SCHEMA_ID): + schema = SCHEMA_REGISTRY.get_schema_by_id(getattr(type_, _BEAM_SCHEMA_ID)) if schema is None: fields = [ schema_pb2.Field( @@ -129,6 +133,7 @@ def typing_to_runner_api(type_): ] type_id = str(uuid4()) schema = schema_pb2.Schema(fields=fields, id=type_id) + setattr(type_, _BEAM_SCHEMA_ID, type_id) SCHEMA_REGISTRY.add(type_, schema) return schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=schema)) @@ -199,7 +204,7 @@ def typing_from_runner_api(fieldtype_proto): [(field.name, typing_from_runner_api(field.type)) for field in schema.fields]) - user_type.id = schema.id + setattr(user_type, _BEAM_SCHEMA_ID, schema.id) # Define a reduce function, otherwise these types can't be pickled # (See BEAM-9574) diff --git a/sdks/python/apache_beam/typehints/schemas_test.py b/sdks/python/apache_beam/typehints/schemas_test.py index 42e7198db4e2..2bf471734019 100644 --- a/sdks/python/apache_beam/typehints/schemas_test.py +++ b/sdks/python/apache_beam/typehints/schemas_test.py @@ -37,6 +37,7 @@ from apache_beam.portability.api import schema_pb2 from apache_beam.typehints.schemas import named_tuple_from_schema +from apache_beam.typehints.schemas import named_tuple_to_schema from apache_beam.typehints.schemas import typing_from_runner_api from apache_beam.typehints.schemas import typing_to_runner_api @@ -285,6 +286,16 @@ def test_generated_class_pickle(self): self.assertEqual(instance, pickle.loads(pickle.dumps(instance))) + def test_user_type_annotated_with_id_after_conversion(self): + MyCuteClass = NamedTuple('MyCuteClass', [ + ('name', unicode), + ]) + self.assertFalse(hasattr(MyCuteClass, '_beam_schema_id')) + + schema = named_tuple_to_schema(MyCuteClass) + self.assertTrue(hasattr(MyCuteClass, '_beam_schema_id')) + self.assertEqual(MyCuteClass._beam_schema_id, schema.id) + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/utils/thread_pool_executor.py b/sdks/python/apache_beam/utils/thread_pool_executor.py index a5443ed5c4b2..c111b9aeff36 100644 --- a/sdks/python/apache_beam/utils/thread_pool_executor.py +++ b/sdks/python/apache_beam/utils/thread_pool_executor.py @@ -134,3 +134,16 @@ def shutdown(self, wait=True): if wait: for worker in self._workers: worker.join() + + +class _SharedUnboundedThreadPoolExecutor(UnboundedThreadPoolExecutor): + def shutdown(self, wait=True): + # Prevent shutting down the shared thread pool + pass + + +_SHARED_UNBOUNDED_THREAD_POOL_EXECUTOR = _SharedUnboundedThreadPoolExecutor() + + +def shared_unbounded_instance(): + return _SHARED_UNBOUNDED_THREAD_POOL_EXECUTOR diff --git a/sdks/python/apache_beam/utils/thread_pool_executor_test.py b/sdks/python/apache_beam/utils/thread_pool_executor_test.py index d9bbae41440b..b9251cad00a1 100644 --- a/sdks/python/apache_beam/utils/thread_pool_executor_test.py +++ b/sdks/python/apache_beam/utils/thread_pool_executor_test.py @@ -30,6 +30,7 @@ # patches unittest.TestCase to be python3 compatible import future.tests.base # pylint: disable=unused-import +from apache_beam.utils import thread_pool_executor from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor @@ -111,6 +112,20 @@ def test_map(self): with self._lock: self.assertEqual(5, len(self._worker_idents)) + def test_shared_shutdown_does_nothing(self): + thread_pool_executor.shared_unbounded_instance().shutdown() + + futures = [] + with thread_pool_executor.shared_unbounded_instance() as executor: + for _ in range(0, 5): + futures.append(executor.submit(self.append_and_sleep, 0.01)) + + for future in futures: + future.result(timeout=10) + + with self._lock: + self.assertEqual(5, len(self._worker_idents)) + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index 154e15b9a6f5..964e46428565 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,4 +17,4 @@ """Apache Beam SDK version information and utilities.""" -__version__ = '2.22.0.dev' +__version__ = '2.23.0.dev' diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index 7af872384c18..f28d6e3b34a0 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -17,7 +17,7 @@ ############################################################################### ARG py_version -FROM python:"${py_version}"-stretch +FROM python:"${py_version}"-buster MAINTAINER "Apache Beam " ARG pull_licenses diff --git a/sdks/python/container/base_image_requirements.txt b/sdks/python/container/base_image_requirements.txt index f3d9b8622118..4c3fd767cb06 100644 --- a/sdks/python/container/base_image_requirements.txt +++ b/sdks/python/container/base_image_requirements.txt @@ -25,18 +25,18 @@ avro==1.8.2;python_version<="2.7" avro-python3==1.8.2;python_version>="3.4" -fastavro==0.21.24 +fastavro==0.23.3 crcmod==1.7 dill==0.3.1.1 -future==0.17.1 +future==0.18.2 futures==3.2.0;python_version<"3.0" grpcio==1.24.3 hdfs==2.5.8 httplib2==0.12.0 mock==2.0.0 oauth2client==3.0.0 -protobuf==3.11.1 -pyarrow==0.15.1 +protobuf==3.12.0 +pyarrow==0.16.0 pydot==1.4.1 pytz==2019.3 pyvcf==0.6.8;python_version<"3.0" @@ -66,8 +66,8 @@ pandas==0.24.2;python_version<="2.7" pandas==0.25.2;python_version>="3.4" protorpc==0.12.0 python-gflags==3.0.6 - -tensorflow==2.1.0 +tensorflow==2.1.0;python_version<="2.7" +tensorflow==2.2.0;python_version>="3.5" pymongo==3.9.0 # Packages needed for testing. diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index 2f7662caf3a7..f759088fbbbf 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -55,6 +55,7 @@ task buildAll { dependsOn ':sdks:python:container:py35:docker' dependsOn ':sdks:python:container:py36:docker' dependsOn ':sdks:python:container:py37:docker' + dependsOn ':sdks:python:container:py38:docker' } artifacts { diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle new file mode 100644 index 000000000000..2a6ec55882d2 --- /dev/null +++ b/sdks/python/container/common.gradle @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') + +description = "Apache Beam :: SDKs :: Python :: Container :: Python ${pythonVersionSuffix} Container" + +configurations { + sdkSourceTarball + sdkHarnessLauncher +} + +dependencies { + sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") + sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") +} + +task copyDockerfileDependencies(type: Copy) { + from configurations.sdkSourceTarball + from file("../base_image_requirements.txt") + into "build/target" + if(configurations.sdkSourceTarball.isEmpty()) { + throw new StopExecutionException(); + } +} + +task copyLicenseScripts(type: Copy){ + from ("../license_scripts") + into "build/target/license_scripts" +} + +task copyLauncherDependencies(type: Copy) { + from configurations.sdkHarnessLauncher + into "build/target/launcher" + if(configurations.sdkHarnessLauncher.isEmpty()) { + throw new StopExecutionException(); + } +} + +docker { + name containerImageName( + name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk", + root: project.rootProject.hasProperty(["docker-repository-root"]) ? + project.rootProject["docker-repository-root"] : + project.docker_image_default_repo_root, + tag: project.rootProject.hasProperty(["docker-tag"]) ? + project.rootProject["docker-tag"] : project.sdk_version) + files "../Dockerfile", "./build" + buildArgs(['py_version': "${project.ext.pythonVersion}", + 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || + project.rootProject.hasProperty(["isRelease"])]) +} + +dockerPrepare.dependsOn copyLauncherDependencies +dockerPrepare.dependsOn copyDockerfileDependencies +dockerPrepare.dependsOn copyLicenseScripts diff --git a/sdks/python/container/license_scripts/dep_urls_py.yaml b/sdks/python/container/license_scripts/dep_urls_py.yaml index 4a2b6f355736..d1e194f0add1 100644 --- a/sdks/python/container/license_scripts/dep_urls_py.yaml +++ b/sdks/python/container/license_scripts/dep_urls_py.yaml @@ -114,7 +114,9 @@ pip_dependencies: license: "https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE" tensorflow-estimator: license: "https://raw.githubusercontent.com/tensorflow/estimator/master/LICENSE" + tensorboard-plugin-wit: + license: "https://raw.githubusercontent.com/PAIR-code/what-if-tool/master/LICENSE" timeloop: license: "https://raw.githubusercontent.com/sankalpjonn/timeloop/master/LICENSE" wget: - license: "https://raw.githubusercontent.com/mirror/wget/master/COPYING" \ No newline at end of file + license: "https://raw.githubusercontent.com/mirror/wget/master/COPYING" diff --git a/sdks/python/container/py2/build.gradle b/sdks/python/container/py2/build.gradle index 4e5848208fa0..6d3c94af3cea 100644 --- a/sdks/python/container/py2/build.gradle +++ b/sdks/python/container/py2/build.gradle @@ -21,55 +21,6 @@ plugins { id 'org.apache.beam.module' } applyDockerNature() +applyPythonNature() -description = "Apache Beam :: SDKs :: Python :: Container :: Python 2 Container" - -configurations { - sdkSourceTarball - sdkHarnessLauncher -} - -dependencies { - sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") - sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") -} - -task copyDockerfileDependencies(type: Copy) { - from configurations.sdkSourceTarball - from file("../base_image_requirements.txt") - into "build/target" - if(configurations.sdkSourceTarball.isEmpty()) { - throw new StopExecutionException(); - } -} - -task copyLicenseScrips(type: Copy){ - from ("../license_scripts") - into "build/target/license_scripts" -} - -task copyLauncherDependencies(type: Copy) { - from configurations.sdkHarnessLauncher - into "build/target/launcher" - if(configurations.sdkHarnessLauncher.isEmpty()) { - throw new StopExecutionException(); - } -} - -docker { - name containerImageName( - name: project.docker_image_default_repo_prefix + "python2.7_sdk", - root: project.rootProject.hasProperty(["docker-repository-root"]) ? - project.rootProject["docker-repository-root"] : - project.docker_image_default_repo_root, - tag: project.rootProject.hasProperty(["docker-tag"]) ? - project.rootProject["docker-tag"] : project.sdk_version) - files "../Dockerfile", "./build" - buildArgs(['py_version': "2.7", - 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || - project.rootProject.hasProperty(["isRelease"])]) -} - -dockerPrepare.dependsOn copyLauncherDependencies -dockerPrepare.dependsOn copyDockerfileDependencies -dockerPrepare.dependsOn copyLicenseScrips +apply from: "../common.gradle" diff --git a/sdks/python/container/py35/build.gradle b/sdks/python/container/py35/build.gradle index ce1fbe91b16a..25aa4f92e14c 100644 --- a/sdks/python/container/py35/build.gradle +++ b/sdks/python/container/py35/build.gradle @@ -21,55 +21,8 @@ plugins { id 'org.apache.beam.module' } applyDockerNature() +applyPythonNature() -description = "Apache Beam :: SDKs :: Python :: Container :: Python 35 Container" +pythonVersion = '3.5' -configurations { - sdkSourceTarball - sdkHarnessLauncher -} - -dependencies { - sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") - sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") -} - -task copyDockerfileDependencies(type: Copy) { - from configurations.sdkSourceTarball - from file("../base_image_requirements.txt") - into "build/target" - if(configurations.sdkSourceTarball.isEmpty()) { - throw new StopExecutionException(); - } -} - -task copyLicenseScrips(type: Copy){ - from ("../license_scripts") - into "build/target/license_scripts" -} - -task copyLauncherDependencies(type: Copy) { - from configurations.sdkHarnessLauncher - into "build/target/launcher" - if(configurations.sdkHarnessLauncher.isEmpty()) { - throw new StopExecutionException(); - } -} - -docker { - name containerImageName( - name: project.docker_image_default_repo_prefix + "python3.5_sdk", - root: project.rootProject.hasProperty(["docker-repository-root"]) ? - project.rootProject["docker-repository-root"] : - project.docker_image_default_repo_root, - tag: project.rootProject.hasProperty(["docker-tag"]) ? - project.rootProject["docker-tag"] : project.sdk_version) - files "../Dockerfile", "./build" - buildArgs(['py_version': "3.5", - 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || - project.rootProject.hasProperty(["isRelease"])]) -} - -dockerPrepare.dependsOn copyLauncherDependencies -dockerPrepare.dependsOn copyDockerfileDependencies -dockerPrepare.dependsOn copyLicenseScrips +apply from: "../common.gradle" diff --git a/sdks/python/container/py36/build.gradle b/sdks/python/container/py36/build.gradle index c5417169c76b..20fe8755bf50 100644 --- a/sdks/python/container/py36/build.gradle +++ b/sdks/python/container/py36/build.gradle @@ -21,55 +21,8 @@ plugins { id 'org.apache.beam.module' } applyDockerNature() +applyPythonNature() -description = "Apache Beam :: SDKs :: Python :: Container :: Python 36 Container" +pythonVersion = '3.6' -configurations { - sdkSourceTarball - sdkHarnessLauncher -} - -dependencies { - sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") - sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") -} - -task copyDockerfileDependencies(type: Copy) { - from configurations.sdkSourceTarball - from file("../base_image_requirements.txt") - into "build/target" - if(configurations.sdkSourceTarball.isEmpty()) { - throw new StopExecutionException(); - } -} - -task copyLicenseScrips(type: Copy){ - from ("../license_scripts") - into "build/target/license_scripts" -} - -task copyLauncherDependencies(type: Copy) { - from configurations.sdkHarnessLauncher - into "build/target/launcher" - if(configurations.sdkHarnessLauncher.isEmpty()) { - throw new StopExecutionException(); - } -} - -docker { - name containerImageName( - name: project.docker_image_default_repo_prefix + "python3.6_sdk", - root: project.rootProject.hasProperty(["docker-repository-root"]) ? - project.rootProject["docker-repository-root"] : - project.docker_image_default_repo_root, - tag: project.rootProject.hasProperty(["docker-tag"]) ? - project.rootProject["docker-tag"] : project.sdk_version) - files "../Dockerfile", "./build" - buildArgs(['py_version': "3.6", - 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || - project.rootProject.hasProperty(["isRelease"])]) -} - -dockerPrepare.dependsOn copyLauncherDependencies -dockerPrepare.dependsOn copyDockerfileDependencies -dockerPrepare.dependsOn copyLicenseScrips +apply from: "../common.gradle" diff --git a/sdks/python/container/py37/build.gradle b/sdks/python/container/py37/build.gradle index 1225332d9535..547163a3514e 100644 --- a/sdks/python/container/py37/build.gradle +++ b/sdks/python/container/py37/build.gradle @@ -21,55 +21,8 @@ plugins { id 'org.apache.beam.module' } applyDockerNature() +applyPythonNature() -description = "Apache Beam :: SDKs :: Python :: Container :: Python 37 Container" +pythonVersion = '3.7' -configurations { - sdkSourceTarball - sdkHarnessLauncher -} - -dependencies { - sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") - sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") -} - -task copyDockerfileDependencies(type: Copy) { - from configurations.sdkSourceTarball - from file("../base_image_requirements.txt") - into "build/target" - if(configurations.sdkSourceTarball.isEmpty()) { - throw new StopExecutionException(); - } -} - -task copyLicenseScrips(type: Copy){ - from ("../license_scripts") - into "build/target/license_scripts" -} - -task copyLauncherDependencies(type: Copy) { - from configurations.sdkHarnessLauncher - into "build/target/launcher" - if(configurations.sdkHarnessLauncher.isEmpty()) { - throw new StopExecutionException(); - } -} - -docker { - name containerImageName( - name: project.docker_image_default_repo_prefix + "python3.7_sdk", - root: project.rootProject.hasProperty(["docker-repository-root"]) ? - project.rootProject["docker-repository-root"] : - project.docker_image_default_repo_root, - tag: project.rootProject.hasProperty(["docker-tag"]) ? - project.rootProject["docker-tag"] : project.sdk_version) - files "../Dockerfile", "./build" - buildArgs(['py_version': "3.7", - 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || - project.rootProject.hasProperty(["isRelease"])]) -} - -dockerPrepare.dependsOn copyLauncherDependencies -dockerPrepare.dependsOn copyDockerfileDependencies -dockerPrepare.dependsOn copyLicenseScrips +apply from: "../common.gradle" diff --git a/sdks/python/container/py38/build.gradle b/sdks/python/container/py38/build.gradle new file mode 100644 index 000000000000..304895a83718 --- /dev/null +++ b/sdks/python/container/py38/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'base' + id 'org.apache.beam.module' +} +applyDockerNature() +applyPythonNature() + +pythonVersion = '3.8' + +apply from: "../common.gradle" diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 2b48344ff149..330759d71d60 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -19,22 +19,23 @@ # This script will be run by Jenkins as a post commit test. In order to run # locally make the following changes: # -# LOCAL_PATH -> Path of tox and virtualenv if you have them already installed. # GCS_LOCATION -> Temporary location to use for service tests. # PROJECT -> Project name to use for dataflow and docker images. +# REGION -> Region name to use for Dataflow # # Execute from the root of the repository: # test Python2 container: ./sdks/python/container/run_validatescontainer.sh python2 # test Python3 container: ./sdks/python/container/run_validatescontainer.sh python35 # test Python3 container: ./sdks/python/container/run_validatescontainer.sh python36 # test Python3 container: ./sdks/python/container/run_validatescontainer.sh python37 +# test Python3 container: ./sdks/python/container/run_validatescontainer.sh python38 -echo "This script must be executed in the root of beam project. Please set LOCAL_PATH, GCS_LOCATION, and PROJECT as desired." +echo "This script must be executed in the root of beam project. Please set GCS_LOCATION, PROJECT and REGION as desired." if [[ $# != 1 ]]; then printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh " printf "\n\tpython_version: [required] Python version used for container build and run tests." - printf " Use 'python2' for Python2, 'python35' for Python3.5, python36 for Python3.6, python37 for Python3.7." + printf " Use 'python2' for Python2, 'python35' for Python3.5, python36 for Python3.6, python37 for Python3.7, python38 for Python3.8." exit 1 fi @@ -46,31 +47,32 @@ GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests} # Project for the container and integration test PROJECT=${PROJECT:-apache-beam-testing} +REGION=${REGION:-us-central1} IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)" # Other variables branched by Python version. if [[ $1 == "python2" ]]; then IMAGE_NAME="${IMAGE_PREFIX}python2.7_sdk" # Use this to create CONTAINER_IMAGE variable. CONTAINER_PROJECT="sdks:python:container:py2" # Use this to build container by Gradle. - GRADLE_PY3_FLAG="" # Use this in Gradle command. PY_INTERPRETER="python" # Use this in virtualenv command. elif [[ $1 == "python35" ]]; then IMAGE_NAME="${IMAGE_PREFIX}python3.5_sdk" # Use this to create CONTAINER_IMAGE variable. CONTAINER_PROJECT="sdks:python:container:py35" # Use this to build container by Gradle. - GRADLE_PY3_FLAG="-Ppython3" # Use this in Gradle command. PY_INTERPRETER="python3.5" # Use this in virtualenv command. elif [[ $1 == "python36" ]]; then IMAGE_NAME="${IMAGE_PREFIX}python3.6_sdk" # Use this to create CONTAINER_IMAGE variable. CONTAINER_PROJECT="sdks:python:container:py36" # Use this to build container by Gradle. - GRADLE_PY3_FLAG="-Ppython3" # Use this in Gradle command. PY_INTERPRETER="python3.6" # Use this in virtualenv command. elif [[ $1 == "python37" ]]; then IMAGE_NAME="${IMAGE_PREFIX}python3.7_sdk" # Use this to create CONTAINER_IMAGE variable. CONTAINER_PROJECT="sdks:python:container:py37" # Use this to build container by Gradle. - GRADLE_PY3_FLAG="-Ppython3" # Use this in Gradle command. PY_INTERPRETER="python3.7" # Use this in virtualenv command. +elif [[ $1 == "python38" ]]; then + IMAGE_NAME="${IMAGE_PREFIX}python3.8_sdk" # Use this to create CONTAINER_IMAGE variable. + CONTAINER_PROJECT="sdks:python:container:py38" # Use this to build container by Gradle. + PY_INTERPRETER="python3.8" # Use this in virtualenv command. else - echo "Must set Python version with one of 'python2', 'python35', 'python36' and 'python37' from commandline." + echo "Must set Python version with one of 'python2', 'python35', 'python36', 'python37' and 'python38' from commandline." exit 1 fi XUNIT_FILE="nosetests-$IMAGE_NAME.xml" @@ -88,7 +90,7 @@ gcloud -v TAG=$(date +%Y%m%d-%H%M%S) CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME echo "Using container $CONTAINER" -./gradlew :$CONTAINER_PROJECT:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG $GRADLE_PY3_FLAG --info +./gradlew :$CONTAINER_PROJECT:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG --info # Verify it exists docker images | grep $TAG diff --git a/sdks/python/mypy.ini b/sdks/python/mypy.ini index 151eebf6b44b..1b24e6cd4ecd 100644 --- a/sdks/python/mypy.ini +++ b/sdks/python/mypy.ini @@ -67,9 +67,6 @@ ignore_errors = true [mypy-apache_beam.coders.*] ignore_errors = true -[mypy-apache_beam.dataframe.*] -ignore_errors = true - [mypy-apache_beam.io.*] ignore_errors = true diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle index f9126da48903..2583d385bc50 100644 --- a/sdks/python/test-suites/tox/py37/build.gradle +++ b/sdks/python/test-suites/tox/py37/build.gradle @@ -35,9 +35,6 @@ lint.dependsOn lintPy37 toxTask "mypyPy37", "py37-mypy" lint.dependsOn mypyPy37 -toxTask "formatter", "py3-yapf-check" -check.dependsOn formatter - apply from: "../common.gradle" // TODO(BEAM-8954): Remove this once tox uses isolated builds. diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle new file mode 100644 index 000000000000..96b02f805c25 --- /dev/null +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for Python 3.8 + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +// Required to setup a Python 3 virtualenv and task names. +pythonVersion = '3.8' + +toxTask "formatter", "py3-yapf-check" +check.dependsOn formatter + +apply from: "../common.gradle" + +// TODO(BEAM-8954): Remove this once tox uses isolated builds. +testPy38Cython.mustRunAfter testPython38, testPy38Cloud diff --git a/sdks/python/test-suites/tox/pycommon/build.gradle b/sdks/python/test-suites/tox/pycommon/build.gradle index 80ad2d4175f0..80342ce990d5 100644 --- a/sdks/python/test-suites/tox/pycommon/build.gradle +++ b/sdks/python/test-suites/tox/pycommon/build.gradle @@ -23,7 +23,7 @@ plugins { id 'org.apache.beam.module' } applyPythonNature() -toxTask "docs", "py37-docs" +toxTask "docs", "py38-docs" assemble.dependsOn docs task preCommitPyCommon() { diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 3b66c28592a0..745ed926907a 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py27,py35,py36,py37,py27-{cloud,cython,lint,lint3},py35-{cloud,cython},py36-{cloud,cython},py37-{cloud,cython,lint,mypy,docs} +envlist = py27,py35,py36,py37,py27-{cloud,cython,lint,lint3},py35-{cloud,cython},py36-{cloud,cython},py37-{cloud,cython,lint,mypy},py38-{cloud,cython,docs} toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -76,6 +76,11 @@ commands = python apache_beam/examples/complete/autocomplete_test.py {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" +[testenv:py38] +commands = + python apache_beam/examples/complete/autocomplete_test.py + {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" + [testenv:py27-cython] # cython tests are only expected to work in linux (2.x and 3.x) # If we want to add other platforms in the future, it should be: @@ -128,6 +133,19 @@ commands = python apache_beam/examples/complete/autocomplete_test.py {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" +[testenv:py38-cython] +# cython tests are only expected to work in linux (2.x and 3.x) +# If we want to add other platforms in the future, it should be: +# `platform = linux2|darwin|...` +# See https://docs.python.org/2/library/sys.html#sys.platform for platform codes +platform = linux +commands = + # TODO(BEAM-8954): Remove this build_ext invocation once local source no longer + # shadows the installed apache_beam. + python setup.py build_ext --inplace + python apache_beam/examples/complete/autocomplete_test.py + {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" + [testenv:py27-cloud] extras = test,gcp,aws commands = @@ -149,6 +167,11 @@ extras = test,gcp,interactive,aws commands = {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" +[testenv:py38-cloud] +extras = test,gcp,interactive,aws +commands = + {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" + [testenv:py27-lint] # Checks for py2 syntax errors deps = @@ -197,10 +220,10 @@ commands = mypy --version python setup.py mypy -[testenv:py37-docs] +[testenv:py38-docs] extras = test,gcp,docs,interactive deps = - Sphinx==1.8.5 + Sphinx==3.0.3 sphinx_rtd_theme==0.4.3 commands = time {toxinidir}/scripts/generate_pydoc.sh diff --git a/settings.gradle b/settings.gradle index 0c642309dfed..058213798eb0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -119,6 +119,7 @@ include ":sdks:java:io:parquet" include ":sdks:java:io:rabbitmq" include ":sdks:java:io:redis" include ":sdks:java:io:solr" +include ":sdks:java:io:snowflake" include ":sdks:java:io:thrift" include ":sdks:java:io:tika" include ":sdks:java:io:xml" @@ -137,6 +138,7 @@ include ":sdks:python:container:py2" include ":sdks:python:container:py35" include ":sdks:python:container:py36" include ":sdks:python:container:py37" +include ":sdks:python:container:py38" include ":sdks:python:test-suites:dataflow:py2" include ":sdks:python:test-suites:dataflow:py35" include ":sdks:python:test-suites:dataflow:py36" @@ -154,6 +156,7 @@ include ":sdks:python:test-suites:tox:py2" include ":sdks:python:test-suites:tox:py35" include ":sdks:python:test-suites:tox:py36" include ":sdks:python:test-suites:tox:py37" +include ":sdks:python:test-suites:tox:py38" include ":vendor:grpc-1_26_0" include ":vendor:bytebuddy-1_10_8" include ":vendor:calcite-1_20_0" diff --git a/website/.gitignore b/website/.gitignore index 8191c0750dc0..bbed591c748a 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -16,7 +16,7 @@ content/ www/node_modules www/dist www/site/resources -www/site/github_samples +www/site/code_samples www/site/_config_branch_repo.toml www/yarn-error.log !www/site/content diff --git a/website/CONTRIBUTE.md b/website/CONTRIBUTE.md index 555fd2a3026e..5176af95dedf 100644 --- a/website/CONTRIBUTE.md +++ b/website/CONTRIBUTE.md @@ -66,7 +66,7 @@ www/ │ │ └── js │   └── themes │ └── docsy -├── build_github_samples.sh +├── build_code_samples.sh ├── check-links.sh # links checker └── package.json ``` @@ -252,20 +252,20 @@ A table markdown here. {{< /table >}} ``` -### Github sample +### Code sample -To retrieve a piece of code in github. +To retrieve a piece of code from Beam project. Usage: ``` -{{< github_sample "/path/to/file" selected_tag >}} +{{< code_sample "path/to/file" selected_tag >}} ``` Example: ``` -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/user_score.py" extract_and_sum_score >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/user_score.py" extract_and_sum_score >}} ``` ### Others diff --git a/website/build.gradle b/website/build.gradle index 235cb8426e6d..7f8e7f69342b 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -104,22 +104,22 @@ task installDependencies(type: Exec) { "${->startDockerContainer.containerId()}", 'yarn', 'install' } -// Run build_github_samples.sh to fetch github content -// which is used by github_sample shortcodes to inject snippets into codeblocks -task buildGithubSamples(type: Exec) { +// Run build_code_samples.sh to fetch Beam project content +// which is used by code_sample shortcodes to inject snippets into codeblocks +task buildCodeSamples(type: Exec) { commandLine 'docker', 'exec', '--workdir', "$dockerSourceDir", - "${->startDockerContainer.containerId()}", 'yarn', 'build_github_samples' + "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples' } task setupDockerContainer(type: Exec) { dependsOn startDockerContainer - finalizedBy initGitSubmodules, installDependencies, buildGithubSamples + finalizedBy initGitSubmodules, installDependencies, buildCodeSamples ext.containerId = { return startDockerContainer.containerId() } // Create the config to point to a GitHub or Colab blob in the repo, e.g. apache/beam/blob/master - commandLine 'docker', 'exec', '-u', 'root', + commandLine 'docker', 'exec', "${->startDockerContainer.containerId()}", '/bin/bash', '-c', """echo '[params]\n branch_repo = "${getBranchRepo()}"' > /tmp/_config_branch_repo.toml""" } diff --git a/website/www/build_github_samples.sh b/website/www/build_code_samples.sh similarity index 75% rename from website/www/build_github_samples.sh rename to website/www/build_code_samples.sh index b47e7d1286cf..a42ba444324b 100755 --- a/website/www/build_github_samples.sh +++ b/website/www/build_code_samples.sh @@ -23,19 +23,21 @@ pushd "${MY_DIR}" &>/dev/null || exit 1 echo "Working directory: ${MY_DIR}" -DIST_DIR=${1:-"./site/github_samples"} +PROJECT_ROOT_REL_PATH=../.. + +DIST_DIR=${1:-"./site/code_samples"} echo "Dist directory: ${DIST_DIR}" CONTENT_DIR=${2:-"./site/content"} -mapfile -t github_urls < <(grep -rh "{{< github_sample" "${CONTENT_DIR}" | sed -e 's/^.*"\(.*\)".*$/\1/g' | sort | uniq | sed 's/\/blob\//\//g' | xargs -n 1 echo) +mapfile -t code_sample_uris < <(grep -rh "{{< code_sample" "${CONTENT_DIR}" | sed -e 's/^.*"\(.*\)".*$/\1/g' | sort | uniq | xargs -n 1 echo) mkdir -pv "${DIST_DIR}" -for url in "${github_urls[@]}" +for uri in "${code_sample_uris[@]}" do - fileName=$(echo "$url" | sed -e 's/\//_/g') - curl -o "$DIST_DIR"/"$fileName" "https://raw.githubusercontent.com$url" + fileName=$(echo "$uri" | sed -e 's/\//_/g') + cp "$PROJECT_ROOT_REL_PATH"/"$uri" "$DIST_DIR"/"$fileName" done popd &>/dev/null || exit 1 diff --git a/website/www/package.json b/website/www/package.json index a5b1cbea637d..186858fb7a8a 100644 --- a/website/www/package.json +++ b/website/www/package.json @@ -5,9 +5,9 @@ "repository": "apache/beam", "license": "MIT", "scripts": { - "build_github_samples": "./build_github_samples.sh", + "build_code_samples": "./build_code_samples.sh", "develop": "cd site && hugo server", - "build": "cross-env HUGO_ENV=production hugo -d ../dist -s site -v", + "build": "cross-env HUGO_ENV=production hugo --minify -d ../dist -s site -v", "start": "hugo -d ../dist -s site -vw" }, "dependencies": {}, diff --git a/website/www/site/config.toml b/website/www/site/config.toml index 9c4bcd179672..5211e364a950 100644 --- a/website/www/site/config.toml +++ b/website/www/site/config.toml @@ -104,7 +104,7 @@ github_project_repo = "https://github.com/apache/beam" [params] description = "Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes." -release_latest = "2.20.0" +release_latest = "2.21.0" # The repository and branch where the files live in Github or Colab. This is used # to serve and stage from your local branch, but publish to the master branch. # e.g. https://github.com/{{< param branch_repo >}}/path/to/notebook.ipynb diff --git a/website/www/site/content/en/blog/beam-2.21.0.md b/website/www/site/content/en/blog/beam-2.21.0.md new file mode 100644 index 000000000000..734642148d48 --- /dev/null +++ b/website/www/site/content/en/blog/beam-2.21.0.md @@ -0,0 +1,97 @@ +--- +title: "Apache Beam 2.21.0" +date: 2020-05-27 00:00:01 -0800 +categories: + - blog +authors: + - ibzib +--- + + +We are happy to present the new 2.21.0 release of Beam. This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/#xxxx-xxxx) for this release. +For more information on changes in 2.21.0, check out the +[detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347143). + +## I/Os +* Python: Deprecated module `apache_beam.io.gcp.datastore.v1` has been removed +as the client it uses is out of date and does not support Python 3 +([BEAM-9529](https://issues.apache.org/jira/browse/BEAM-9529)). +Please migrate your code to use +[apache_beam.io.gcp.datastore.**v1new**](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.datastore.v1new.datastoreio.html). +See the updated +[datastore_wordcount](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py) +for example usage. +* Python SDK: Added integration tests and updated batch write functionality for Google Cloud Spanner transform ([BEAM-8949](https://issues.apache.org/jira/browse/BEAM-8949)). + +## New Features / Improvements +* Python SDK will now use Python 3 type annotations as pipeline type hints. +([#10717](https://github.com/apache/beam/pull/10717)) + + If you suspect that this feature is causing your pipeline to fail, calling + `apache_beam.typehints.disable_type_annotations()` before pipeline creation + will disable is completely, and decorating specific functions (such as + `process()`) with `@apache_beam.typehints.no_annotations` will disable it + for that function. + + More details will be in + [Ensuring Python Type Safety](https://beam.apache.org/documentation/sdks/python-type-safety/) + and an upcoming + [blog post](https://beam.apache.org/blog/python/typing/2020/03/06/python-typing.html). + +* Java SDK: Introducing the concept of options in Beam Schema’s. These options add extra +context to fields and schemas. This replaces the current Beam metadata that is present +in a FieldType only, options are available in fields and row schemas. Schema options are +fully typed and can contain complex rows. *Remark: Schema aware is still experimental.* +([BEAM-9035](https://issues.apache.org/jira/browse/BEAM-9035)) +* Java SDK: The protobuf extension is fully schema aware and also includes protobuf option +conversion to beam schema options. *Remark: Schema aware is still experimental.* +([BEAM-9044](https://issues.apache.org/jira/browse/BEAM-9044)) +* Added ability to write to BigQuery via Avro file loads (Python) ([BEAM-8841](https://issues.apache.org/jira/browse/BEAM-8841)) + + By default, file loads will be done using JSON, but it is possible to + specify the temp_file_format parameter to perform file exports with AVRO. + AVRO-based file loads work by exporting Python types into Avro types, so + to switch to Avro-based loads, you will need to change your data types + from Json-compatible types (string-type dates and timestamp, long numeric + values as strings) into Python native types that are written to Avro + (Python's date, datetime types, decimal, etc). For more information + see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-avro#avro_conversions. +* Added integration of Java SDK with Google Cloud AI VideoIntelligence service +([BEAM-9147](https://issues.apache.org/jira/browse/BEAM-9147)) +* Added integration of Java SDK with Google Cloud AI natural language processing API +([BEAM-9634](https://issues.apache.org/jira/browse/BEAM-9634)) +* `docker-pull-licenses` tag was introduced. Licenses/notices of third party dependencies will be added to the docker images when `docker-pull-licenses` was set. + The files are added to `/opt/apache/beam/third_party_licenses/`. + By default, no licenses/notices are added to the docker images. ([BEAM-9136](https://issues.apache.org/jira/browse/BEAM-9136)) + + +## Breaking Changes + +* Dataflow runner now requires the `--region` option to be set, unless a default value is set in the environment ([BEAM-9199](https://issues.apache.org/jira/browse/BEAM-9199)). See [here](https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) for more details. +* HBaseIO.ReadAll now requires a PCollection of HBaseIO.Read objects instead of HBaseQuery objects ([BEAM-9279](https://issues.apache.org/jira/browse/BEAM-9279)). +* ProcessContext.updateWatermark has been removed in favor of using a WatermarkEstimator ([BEAM-9430](https://issues.apache.org/jira/browse/BEAM-9430)). +* Coder inference for PCollection of Row objects has been disabled ([BEAM-9569](https://issues.apache.org/jira/browse/BEAM-9569)). +* Go SDK docker images are no longer released until further notice. + +## Deprecations +* Java SDK: Beam Schema FieldType.getMetadata is now deprecated and is replaced by the Beam +Schema Options, it will be removed in version `2.23.0`. ([BEAM-9704](https://issues.apache.org/jira/browse/BEAM-9704)) +* The `--zone` option in the Dataflow runner is now deprecated. Please use `--worker_zone` instead. ([BEAM-9716](https://issues.apache.org/jira/browse/BEAM-9716)) + + +## List of Contributors + +According to git shortlog, the following people contributed to the 2.21.0 release. Thank you to all contributors! + +Aaron Meihm, Adrian Eka, Ahmet Altay, AldairCoronel, Alex Van Boxel, Alexey Romanenko, Andrew Crites, Andrew Pilloud, Ankur Goenka, Badrul (Taki) Chowdhury, Bartok Jozsef, Boyuan Zhang, Brian Hulette, brucearctor, bumblebee-coming, Chad Dombrova, Chamikara Jayalath, Chie Hayashida, Chris Gorgolewski, Chuck Yang, Colm O hEigeartaigh, Curtis "Fjord" Hawthorne, Daniel Mills, Daniel Oliveira, David Yan, Elias Djurfeldt, Emiliano Capoccia, Etienne Chauchot, Fernando Diaz, Filipe Regadas, Gleb Kanterov, Hai Lu, Hannah Jiang, Harch Vardhan, Heejong Lee, Henry Suryawirawan, Hk-tang, Ismaël Mejía, Jacoby, Jan Lukavský, Jeroen Van Goey, jfarr, Jozef Vilcek, Kai Jiang, Kamil Wasilewski, Kenneth Knowles, KevinGG, Kyle Weaver, Kyoungha Min, Luke Cwik, Maximilian Michels, Michal Walenia, Ning Kang, Pablo Estrada, paul fisher, Piotr Szuberski, Reuven Lax, Robert Bradshaw, Robert Burke, Rose Nguyen, Rui Wang, Sam Rohde, Sam Whittle, Spoorti Kundargi, Steve Koonce, sunjincheng121, Ted Yun, Tesio, Thomas Weise, Tomo Suzuki, Udi Meiri, Valentyn Tymofieiev, Vasu Nori, Yichi Zhang, yoshiki.obata, Yueyang Qiu diff --git a/website/www/site/content/en/blog/beam-summit-digital-2020.md b/website/www/site/content/en/blog/beam-summit-digital-2020.md index ed0853c58d58..dfbe9fdd7c1c 100644 --- a/website/www/site/content/en/blog/beam-summit-digital-2020.md +++ b/website/www/site/content/en/blog/beam-summit-digital-2020.md @@ -1,6 +1,6 @@ --- layout: post -title: "Beam Digital Summit is Coming, and it's Coming Fast!" +title: "Beam Summit Digital Is Coming - Register Now!" date: 2020-05-08 00:00:01 -0800 aliases: - /blog/2020/05/08/beam-summit-digital-2020.html @@ -23,13 +23,13 @@ See the License for the specific language governing permissions and limitations under the License. --> -As some of you are already aware, the 2020 edition of the Beam Summit will be completely **digital and free**. And as you would expect from the Apache Beam community, it is coming in fast; the Beam Digital Summit will be from **June 15th to 19th**, one moth away! The conference will be spread across the course of one week with a couple of hours of program each day. +As some of you are already aware, the 2020 edition of the Beam Summit will be completely **digital and free**. Beam Summit Digital will take place from **August 24th to 28th**. The conference will be spread across the course of one week with a couple of hours of program each day. Beam Summit Digital 2020 + src="/images/blog/beamsummit/beamsummit-digital-2020.png" + alt="Beam Summit Digital 2020, August 24-28"> While we would have loved to see all of you in person, we have to accept that 2020 will not be the year for that. So, we are looking at this as an opportunity to have a bigger and more inclusive event, where people who would normally not be able to travel to the summit will now be able to join, learn and share with the rest of the community. @@ -43,7 +43,7 @@ So, what we want to say with this is: We will have a great event! And if you hav As all things Beam, this is a community effort. The door is open for participation: -1. Submit a proposal to talk. Please check out the **[Call for Papers](https://sessionize.com/beam-digital-summit-2020/)** and submit a talk. The deadline for submissions is _May 20th_! +1. Submit a proposal to talk. Please check out the **[Call for Papers](https://sessionize.com/beam-digital-summit-2020/)** and submit a talk. The deadline for submissions is _June 15th_! 2. Register to join as an attendee. Registration is now open at the **[registration page](https://crowdcast.io/e/beamsummit)**. Registration is free! 3. Consider sponsoring the event. If your company is interested in engaging with members of the community please check out our [sponsoring prospectus](https://drive.google.com/open?id=1EbijvZKpkWwWyMryLY9sJfyZzZk1k44v). 4. Help us get the word out. Please make sure to let your colleagues and friends in the data engineering field (and beyond!) know about the Beam Summit. diff --git a/website/www/site/content/en/blog/python-typing.md b/website/www/site/content/en/blog/python-typing.md new file mode 100644 index 000000000000..7982c19fe9e4 --- /dev/null +++ b/website/www/site/content/en/blog/python-typing.md @@ -0,0 +1,136 @@ +--- +layout: post +title: "Python SDK Typing Changes" +date: 2020-05-28 00:00:01 -0800 +categories: + - blog + - python + - typing +authors: + - chadrik + - udim +--- + + +Beam Python has recently increased its support and integration of Python 3 type +annotations for improved code clarity and type correctness checks. +Read on to find out what's new. + + + +Python supports type annotations on functions (PEP 484). Static type checkers, +such as mypy, are used to verify adherence to these types. +For example: +``` +def f(v: int) -> int: + return v[0] +``` +Running mypy on the above code will give the error: +`Value of type "int" is not indexable`. + +We've recently made changes to Beam in 2 areas: + +Adding type annotations throughout Beam. Type annotations make a large and +sophisticated codebase like Beam easier to comprehend and navigate in your +favorite IDE. + +Second, we've added support for Python 3 type annotations. This allows SDK +users to specify a DoFn's type hints in one place. +We've also expanded Beam's support of `typing` module types. + +For more background see: +[Ensuring Python Type Safety](https://beam.apache.org/documentation/sdks/python-type-safety/). + +# Beam Is Typed + +In tandem with the new type annotation support within DoFns, we've invested a +great deal of time adding type annotations to the Beam python code itself. +With this in place, we have begun using mypy, a static type +checker, as part of Beam's code review process, which ensures higher quality +contributions and fewer bugs. +The added context and insight that type annotations add throughout Beam is +useful for all Beam developers, contributors and end users alike, but +it is especially beneficial for developers who are new to the project. +If you use an IDE that understands type annotations, it will provide richer +type completions and warnings than before. +You'll also be able to use your IDE to inspect the types of Beam functions and +transforms to better understand how they work, which will ease your own +development. +Finally, once Beam is fully annotated, end users will be able to benefit from +the use of static type analysis on their own pipelines and custom transforms. + +# New Ways to Annotate + +## Python 3 Syntax Annotations + +Coming in Beam 2.21 (BEAM-8280), you will be able to use Python annotation +syntax to specify input and output types. + +For example, this new form: +``` +class MyDoFn(beam.DoFn): + def process(self, element: int) -> typing.Text: + yield str(element) +``` +is equivalent to this: +``` +@apache_beam.typehints.with_input_types(int) +@apache_beam.typehints.with_output_types(typing.Text) +class MyDoFn(beam.DoFn): + def process(self, element): + yield str(element) +``` + +One of the advantages of the new form is that you may already be using it +in tandem with a static type checker such as mypy, thus getting additional +runtime type checking for free. + +This feature will be enabled by default, and there will be 2 mechanisms in +place to disable it: +1. Calling `apache_beam.typehints.disable_type_annotations()` before pipeline +construction will disable the new feature completely. +1. Decorating a function with `@apache_beam.typehints.no_annotations` will +tell Beam to ignore annotations for it. + +Uses of Beam's `with_input_type`, `with_output_type` methods and decorators will +still work and take precedence over annotations. + +### Sidebar + +You might ask: couldn't we use mypy to type check Beam pipelines? +There are several reasons why this is not the case. +- Pipelines are constructed at runtime and may depend on information that is +only known at that time, such as a config file or database table schema. +- PCollections don't have the necessary type information, so mypy sees them as +effectively containing any element type. +This may change in in the future. +- Transforms using lambdas (ex: `beam.Map(lambda x: (1, x)`) cannot be +annotated properly using PEP 484. +However, Beam does a best-effort attempt to analyze the output type +from the bytecode. + +## Typing Module Support + +Python's [typing](https://docs.python.org/3/library/typing.html) module defines +types used in type annotations. This is what we call "native" types. +While Beam has its own typing types, it also supports native types. +While both Beam and native types are supported, for new code we encourage using +native typing types. Native types have as these are supported by additional tools. + +While working on Python 3 annotations syntax support, we've also discovered and +fixed issues with native type support. There may still be bugs and unsupported +native types. Please +[let us know](https://beam.apache.org/community/contact-us/) if you encounter +issues. diff --git a/website/www/site/content/en/community/powered-by.md b/website/www/site/content/en/community/powered-by.md new file mode 100644 index 000000000000..ad22039388b9 --- /dev/null +++ b/website/www/site/content/en/community/powered-by.md @@ -0,0 +1,25 @@ +--- +title: 'Powered by Apache Beam' +--- + +# Projects and Products Powered by Apache Beam + +To add yourself to the list, please open a [pull request](https://github.com/apache/beam/edit/master/website/www/site/content/en/community/powered_by.md) adding your organization name, URL, a list of which Beam components you are using, and a short description of your use case. + +* **[Cloud Dataflow](https://cloud.google.com/dataflow):** Google Cloud Dataflow is a fully managed service for executing + Apache Beam pipelines within the Google Cloud Platform ecosystem. +* **[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx):** TensorFlow Extended (TFX) is an end-to-end platform + for deploying production ML pipelines. + diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 95c95c2272bb..441a2167901e 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -171,7 +171,6 @@ Configure access to the [Apache Nexus repository](https://repository.apache.org/ -__NOTE__: make sure the XML you end up with matches the structure above. #### Submit your GPG public key into MIT PGP Public Key Server In order to make yourself have right permission to stage java artifacts in Apache Nexus staging repository, @@ -398,7 +397,7 @@ There are 2 ways to perform this verification, either running automation script( Jenkins job `beam_Release_Gradle_Build` basically run `./gradlew build -PisRelease`. This only verifies that everything builds with unit tests passing. -You can refer to [this script](https://gist.github.com/Ardagan/13e6031e8d1c9ebbd3029bf365c1a517) to mass-comment on PR. +You can use [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) to mass-comment on PR. #### Verify the build succeeds @@ -406,7 +405,6 @@ You can refer to [this script](https://gist.github.com/Ardagan/13e6031e8d1c9ebbd 1. Check the build result. 2. If build failed, scan log will contain all failures. 3. You should stabilize the release branch until release build succeeded. - 4. The script will output a set of Jenkins phrases to enter in the created PR. There are some projects that don't produce the artifacts, e.g. `beam-test-tools`, you may be able to ignore failures there. @@ -583,189 +581,43 @@ For this step, we recommend you using automation script to create a RC, but you 1. Stage source release into dist.apache.org dev [repo](https://dist.apache.org/repos/dist/dev/beam/). 1. Stage,sign and hash python binaries into dist.apache.ord dev repo python dir 1. Stage SDK docker images to [docker hub Apache organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image). - 1. Create a PR to update beam and beam-site, changes includes: + 1. Create a PR to update beam-site, changes includes: * Copy python doc into beam-site * Copy java doc into beam-site - * Update release version into [_config.yml](https://github.com/apache/beam/blob/master/website/_config.yml). #### Tasks you need to do manually - 1. Add new release into `website/src/get-started/downloads.md`. - 1. Update last release download links in `website/src/get-started/downloads.md`. - 1. Update `website/src/.htaccess` to redirect to the new version. - 1. Build and stage python wheels. + 1. Verify the script worked. + 1. Verify that the source and Python binaries are present in [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam). + 1. Verify Docker images are published. How to find images: + 1. Visit [https://hub.docker.com/u/apache](https://hub.docker.com/search?q=apache%2Fbeam&type=image) + 2. Visit each repository and navigate to *tags* tab. + 3. Verify images are pushed with tags: ${RELEASE}_rc{RC_NUM} + 1. Verify that third party licenses are included in Docker containers by logging in to the images. + - For Python SDK images, there should be around 80 ~ 100 dependencies. + Please note that dependencies for the SDKs with different Python versions vary. + Need to verify all Python images by replacing `${ver}` with each supported Python version `X.Y`. + ``` + docker run -it --entrypoint=/bin/bash apache/beam_python${ver}_sdk:${RELEASE}_rc{RC_NUM} + ls -al /opt/apache/beam/third_party_licenses/ | wc -l + ``` + - For Java SDK images, there should be around 1400 dependencies. + ``` + docker run -it --entrypoint=/bin/bash apache/beam_java_sdk:${RELEASE}_rc{RC_NUM} + ls -al /opt/apache/beam/third_party_licenses/ | wc -l + ``` 1. Publish staging artifacts - 1. Go to the staging repo to close the staging repository on [Apache Nexus](https://repository.apache.org/#stagingRepositories). + 1. Log in to the [Apache Nexus](https://repository.apache.org/#stagingRepositories) website. + 1. Navigate to Build Promotion -> Staging Repositories (in the left sidebar). + 1. Select repository `orgapachebeam-NNNN`. + 1. Click the Close button. 1. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. - - -### (Alternative) Run all steps manually - -#### Build and stage Java artifacts with Gradle - -Set up a few environment variables to simplify the commands that follow. These identify the release candidate being built, and the branch where you will stage files. Start with `RC_NUM` equal to `1` and increment it for each candidate. - - RC_NUM=1 - -Make sure your git config will maintain your account: - - git config credential.helper store - -Use Gradle release plugin to build the release artifacts, and push code and -release tag to the origin repository (this would be the Apache Beam repo): - - ./gradlew release -Prelease.newVersion=${RELEASE}-SNAPSHOT \ - -Prelease.releaseVersion=${RELEASE}-RC${RC_NUM} \ - -Prelease.useAutomaticVersion=true --info --no-daemon - -Use Gradle publish plugin to stage these artifacts on the Apache Nexus repository, as follows: - - ./gradlew publish -PisRelease --no-parallel --no-daemon - -Review all staged artifacts. They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. Carefully review any new artifacts. - -Close the staging repository on Apache Nexus. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. - -#### Stage source release on dist.apache.org - -Attention: Only committer has permissions to perform following steps. - -Copy the source release to the dev repository of `dist.apache.org`. - -1. If you have not already, check out the Beam section of the `dev` repository on `dist.apache.org` via Subversion. In a fresh directory: - - svn co https://dist.apache.org/repos/dist/dev/beam - -1. Make a directory for the new release: - - mkdir beam/${RELEASE} - cd beam/${RELEASE} - -1. Download source zip from GitHub: - - wget https://github.com/apache/beam/archive/release-${RELEASE}.zip \ - -O apache-beam-${RELEASE}-source-release.zip - -1. Create hashes and sign the source distribution: - - gpg --armor --detach-sig apache-beam-${RELEASE}-source-release.zip - sha512sum apache-beam-${RELEASE}-source-release.zip > apache-beam-${RELEASE}-source-release.zip.sha512 - -1. Add and commit all the files. - - svn add beam/${RELEASE} - svn commit - -1. Verify that files are [present](https://dist.apache.org/repos/dist/dev/beam). - -#### Stage python binaries on dist.apache.org - -Build python binaries in release branch in sdks/python dir. - - pip install -r build-requirements.txt - python setup.py sdist --format=zip - cd dist - cp apache-beam-${RELEASE}.zip staging/apache-beam-${RELEASE}-python.zip - cd staging - -Create hashes and sign the binaries - - gpg --armor --detach-sig apache-beam-${RELEASE}-python.zip - sha512sum apache-beam-${RELEASE}-python.zip > apache-beam-${RELEASE}-python.zip.sha512 - -Staging binaries - - svn co https://dist.apache.org/repos/dist/dev/beam - cd beam/${RELEASE} - svn add * - svn commit - -Verify that files are [present](https://dist.apache.org/repos/dist/dev/beam). - -#### Stage SDK images on hub.docker.com -* Build Python images and push to DockerHub. - -``` -./gradlew :sdks:python:container:buildAll -Pdocker-pull-licenses -Pdocker-tag=${RELEASE}_rc{RC_NUM} -``` - -Verify that third party licenses are included by logging in to the images. For Python SDK images, there should be around 80 ~ 100 dependencies. -Please note that dependencies for the SDKs with different Python versions vary. -Need to verify all Python images by replacing `${ver}` in the following command to `python2.7, python3.5, python3.6, python3.7`. - -``` -docker run -it --entrypoint=/bin/bash apache/beam_${ver}_sdk:${RELEASE}_rc{RC_NUM} -ls -al /opt/apache/beam/third_party_licenses/ | wc -l -``` - -After verifying the third party licenses are included correctly, push the images to DockerHub. -``` -PYTHON_VER=("python2.7" "python3.5" "python3.6" "python3.7") -for ver in "${PYTHON_VER[@]}"; do - docker push apache/beam_${ver}_sdk:${RELEASE}_rc{RC_NUM} & -done -``` - -* Build Java images and push to DockerHub. - -``` -./gradlew :sdks:java:container:docker -Pdocker-pull-licenses -Pdocker-tag=${RELEASE}_rc{RC_NUM} -``` - -Verify that third party licenses are included by logging in to the images. For Java SDK images, there should be around 1400 dependencies. -``` -docker run -it --entrypoint=/bin/bash apache/beam_java_sdk:${RELEASE}_rc{RC_NUM} -ls -al /opt/apache/beam/third_party_licenses/ | wc -l -``` - -After verifying the third party licenses are included correctly, push the images to DockerHub. -``` -docker push apache/beam_java_sdk:${RELEASE}_rc{RC_NUM} -``` - -* Build Flink job server images and push to DockerHub. - -``` -FLINK_VER=("1.8" "1.9" "1.10") -for ver in "${FLINK_VER[@]}"; do - ./gradlew ":runners:flink:${ver}:job-server-container:dockerPush" -Pdocker-tag="${RELEASE}_rc${RC_NUM}" -done -``` - -* Build Spark job server image and push to DockerHub. - -``` -./gradlew ":runners:spark:job-server:container:dockerPush" -Pdocker-tag="${RELEASE}_rc${RC_NUM}" -``` - -Clean up images from local - -``` -for ver in "${PYTHON_VER[@]}"; do - docker rmi -f apache/beam_${ver}_sdk:${RELEASE}_rc{RC_NUM} -done -docker rmi -f apache/beam_java_sdk:${RELEASE}_rc{RC_NUM} -for ver in "${FLINK_VER[@]}"; do - docker rmi -f "apache/beam_flink${ver}_job_server:${RELEASE}_rc${RC_NUM}" -done -docker rmi -f "apache/beam_spark_job_server:${RELEASE}_rc${RC_NUM}" -``` - -How to find images: -1. Visit [https://hub.docker.com/u/apache](https://hub.docker.com/search?q=apache%2Fbeam&type=image) -2. Visit each repository and navigate to *tags* tab. -3. Verify images are pushed with tags: ${RELEASE}_rc{RC_NUM} - -### Build and stage python wheels - -There is a wrapper repo [beam-wheels](https://github.com/apache/beam-wheels) to help build python wheels. - -If you are interested in how it works, please refer to the [structure section](https://github.com/apache/beam-wheels#structure). - -Please follow the [user guide](https://github.com/apache/beam-wheels#user-guide) to build python wheels. - -Once all python wheels have been staged [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/), -please run [./sign_hash_python_wheels.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/sign_hash_python_wheels.sh) to sign and hash python wheels. - + 1. Review all staged artifacts on https://repository.apache.org/content/repositories/orgapachebeam-NNNN/. They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. Carefully review any new artifacts. + 1. Build and stage python wheels. + - There is a wrapper repo [beam-wheels](https://github.com/apache/beam-wheels) to help build python wheels. + - If you are interested in how it works, please refer to the [structure section](https://github.com/apache/beam-wheels#structure). + - Please follow the [user guide](https://github.com/apache/beam-wheels#user-guide) to build python wheels. + - Once all python wheels have been staged to GCS, + please run [./sign_hash_python_wheels.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/sign_hash_python_wheels.sh), which copies the wheels along with signatures and hashes to [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/). ********** @@ -818,31 +670,22 @@ to the Beam website, usually within an hour. **PR 1: apache/beam-site** This pull request is against the `apache/beam-site` repo, on the `release-docs` -branch. - -* Add the new Javadoc to [SDK API Reference page](https://beam.apache.org/releases/javadoc/) page, as follows: - * Unpack the Maven artifact `org.apache.beam:beam-sdks-java-javadoc` into some temporary location. Call this `${JAVADOC_TMP}`. - * Copy the generated Javadoc into the website repository: `cp -r ${JAVADOC_TMP} javadoc/${RELEASE}`. -* Add the new Pydoc to [SDK API Reference page](https://beam.apache.org/releases/pydoc/) page, as follows: - * Copy the generated Pydoc into the website repository: `cp -r ${PYDOC_ROOT} pydoc/${RELEASE}`. - * Remove `.doctrees` directory. -* Stage files using: `git add --all javadoc/ pydoc/`. +branch ([example](https://github.com/apache/beam-site/pull/603)). +It is created by `build_release_candidate.sh` (see above). **PR 2: apache/beam** -This pull request is against the `apache/beam` repo, on the `master` branch. +This pull request is against the `apache/beam` repo, on the `master` branch ([example](https://github.com/apache/beam/pull/11727)). -* Update the `release_latest` version flag in `/website/_config.yml`, and list - the new release in `/website/src/get-started/downloads.md`, linking to the - source code download and the Release Notes in JIRA. -* Update the `RedirectMatch` rule in - [/website/src/.htaccess](https://github.com/apache/beam/blob/master/website/src/.htaccess) - to point to the new release. See file history for examples. +* Update release version in `website/www/site/config.toml`. +* Add new release in `website/www/site/content/en/get-started/downloads.md`. + * Download links will not work until the release is finalized. +* Update `website/www/site/static/.htaccess` to redirect to the new version. ### Blog post -Write a blog post similar to https://beam.apache.org/blog/2019/08/22/beam-2.15.0.html +Write a blog post similar to [beam-2.20.0.md](https://github.com/apache/beam/blob/master/website/www/site/content/en/blog/beam-2.20.0.md). - Update `CHANGES.md` by adding a new section for the next release. - Copy the changes for the current release from `CHANGES.md` to the blog post and edit as necessary. @@ -856,53 +699,53 @@ all major features and bug fixes, and all known issues. Template: ``` - We are happy to present the new {$RELEASE_VERSION} release of Beam. This release includes both improvements and new functionality. - See the [download page](/get-started/downloads/{$DOWNLOAD_ANCHOR}) for this release. - For more information on changes in {$RELEASE_VERSION}, check out the - [detailed release notes]({$JIRA_RELEASE_NOTES}). +We are happy to present the new {$RELEASE_VERSION} release of Beam. This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/{$DOWNLOAD_ANCHOR}) for this release. +For more information on changes in {$RELEASE_VERSION}, check out the +[detailed release notes]({$JIRA_RELEASE_NOTES}). - ## Highlights +## Highlights - * New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). - * New highly anticipated feature Y added to JavaSDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + * New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + * New highly anticipated feature Y added to JavaSDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). - {$TOPICS e.g.:} - ### I/Os - * Support for X source added (Java) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). - {$TOPICS} +{$TOPICS e.g.:} +### I/Os +* Support for X source added (Java) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +{$TOPICS} - ### New Features / Improvements +### New Features / Improvements - * X feature added (Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). - * Y feature added (Java) [BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y). +* X feature added (Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* Y feature added (Java) [BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y). - ### Breaking Changes +### Breaking Changes - * X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). - * Y behavior was changed ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* Y behavior was changed ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). - ### Deprecations +### Deprecations - * X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). - ### Bugfixes +### Bugfixes - * Fixed X (Python) ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-X)). - * Fixed Y (Java) ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). +* Fixed X (Python) ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-X)). +* Fixed Y (Java) ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). - ### Known Issues +### Known Issues - * {$KNOWN_ISSUE_1} - * {$KNOWN_ISSUE_2} - * See a full list of open [issues that affect](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20affectedVersion%20%3D%20{$RELEASE}%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC) this version. +* {$KNOWN_ISSUE_1} +* {$KNOWN_ISSUE_2} +* See a full list of open [issues that affect](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20affectedVersion%20%3D%20{$RELEASE}%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC) this version. - ## List of Contributors +## List of Contributors - According to git shortlog, the following people contributed to the 2.XX.0 release. Thank you to all contributors! +According to git shortlog, the following people contributed to the 2.XX.0 release. Thank you to all contributors! - ${CONTRIBUTORS} - ``` +${CONTRIBUTORS} +``` #### Checklist to proceed to the next step @@ -1272,24 +1115,23 @@ please follow [the guide](https://help.github.com/articles/creating-a-personal-a ### Deploy Python artifacts to PyPI -1. Download everything from https://dist.apache.org/repos/dist/dev/beam/2.14.0/python/ ; -2. Keep only things that you see in https://pypi.org/project/apache-beam/#files , e.g. `.zip`, `.whl`, - delete the `.asc`, `.sha512`; -3. Upload the new release `twine upload *` from the directory with the `.zip` and `.whl` files; - -[Installing twine](https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives): `pip install twine`. You can install twine under [virtualenv](https://virtualenv.pypa.io/en/latest/) if preferred. - +* Script: [deploy_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_pypi.sh) +* Usage +``` +./beam/release/src/main/scripts/deploy_pypi.sh +``` +* Verify that the files at https://pypi.org/project/apache-beam/#files are correct. +All wheels should be published, in addition to the zip of the release source. +(Signatures and hashes do _not_ need to be uploaded.) ### Deploy source release to dist.apache.org Copy the source release from the `dev` repository to the `release` repository at `dist.apache.org` using Subversion. -Move last release artifacts from `dist.apache.org` to `archive.apache.org` using Subversion. Then update download address for last release version, [example PR](https://github.com/apache/beam-site/pull/478). +Move last release artifacts from `dist.apache.org` to `archive.apache.org` using Subversion. Make sure to change these links on the website ([example](https://github.com/apache/beam/pull/11727)). __NOTE__: Only PMC members have permissions to do it, ping [dev@](mailto:dev@beam.apache.org) for assitance; -Make sure the download address for last release version is upldaed, [example PR](https://github.com/apache/beam-site/pull/478). - ### Deploy SDK docker images to DockerHub * Script: [publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh) * Usage diff --git a/website/www/site/content/en/documentation/dsls/sql/overview.md b/website/www/site/content/en/documentation/dsls/sql/overview.md index fc89197ef884..f91037b3e9da 100644 --- a/website/www/site/content/en/documentation/dsls/sql/overview.md +++ b/website/www/site/content/en/documentation/dsls/sql/overview.md @@ -38,7 +38,7 @@ There are two additional concepts you need to know to use SQL in your pipeline: - [SqlTransform](https://beam.apache.org/releases/javadoc/{{< param release_latest >}}/index.html?org/apache/beam/sdk/extensions/sql/SqlTransform.html): the interface for creating `PTransforms` from SQL queries. - [Row](https://beam.apache.org/releases/javadoc/{{< param release_latest >}}/index.html?org/apache/beam/sdk/values/Row.html): the type of elements that Beam SQL operates on. A `PCollection` plays the role of a table. -{{< param release_latest >}} + ## Walkthrough The [SQL pipeline walkthrough](/documentation/dsls/sql/walkthrough) works through how to use Beam SQL with example code. diff --git a/website/www/site/content/en/documentation/io/built-in.md b/website/www/site/content/en/documentation/io/built-in.md index 481c532327f9..e40884e1e798 100644 --- a/website/www/site/content/en/documentation/io/built-in.md +++ b/website/www/site/content/en/documentation/io/built-in.md @@ -19,87 +19,11 @@ limitations under the License. This table contains the currently available I/O transforms. -Consult the [Programming Guide I/O section](/documentation/programming-guide#pipeline-io) for general usage instructions, and see the javadoc/pydoc for the particular I/O transforms. +Consult the [Programming Guide I/O section](/documentation/programming-guide#pipeline-io) for general usage instructions. +{{< language-switcher java py go >}} - - - - - - - - - - - - - - - - - - - - - - - - - -
LanguageFile-basedMessagingDatabase
Java -

Beam Java supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.

-

FileIO (general-purpose reading, writing, and matching of files)

-

AvroIO

-

TextIO

-

TFRecordIO

-

XmlIO

-

TikaIO

-

ParquetIO

-

ThriftIO

-
-

Amazon Kinesis

-

AMQP

-

Apache Kafka

-

Google Cloud Pub/Sub

-

JMS

-

MQTT

-

RabbitMqIO

-

SqsIO

-
-

Apache Cassandra

-

Apache Hadoop Input/Output Format

-

Apache HBase

-

Apache Hive (HCatalog)

-

Apache Kudu

-

Apache Solr

-

Elasticsearch (v2.x, v5.x, v6.x)

-

Google BigQuery

-

Google Cloud Bigtable

-

Google Cloud Datastore

-

Google Cloud Spanner

-

JDBC

-

MongoDB

-

Redis

-
Python/Batch -

Beam Python supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.

-

avroio

-

parquetio.py

-

textio

-

tfrecordio

-

vcfio

-
- -

Google BigQuery

-

Google Cloud Datastore

-

Google Cloud Bigtable (Write)

-

MongoDB

-
Python/Streaming - -

Google Cloud Pub/Sub

-
-

Google BigQuery (sink only)

-
+{{< io-matrix >}} # In-Progress I/O Transforms @@ -137,6 +61,10 @@ This table contains I/O transforms that are currently planned or in-progress. St Neo4jJava BEAM-1857 + + Pub/Sub LiteJava + BEAM-10114 + RestIOJava BEAM-1946 diff --git a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md index d549f5569fed..16fa33e41819 100644 --- a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md +++ b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md @@ -103,11 +103,11 @@ To specify a table with a string, use the format table name. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpec >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpec >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec >}} {{< /highlight >}} You can also omit `project_id` and use the `[dataset_id].[table_id]` format. If @@ -120,11 +120,11 @@ you omit the project ID, Beam uses the default project ID from your {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpecWithoutProject >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpecWithoutProject >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec_without_project >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec_without_project >}} {{< /highlight >}} #### Using a TableReference @@ -133,11 +133,11 @@ To specify a table with a `TableReference`, create a new `TableReference` using the three parts of the BigQuery table name. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpecObject >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTableSpecObject >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec_object >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_table_spec_object >}} {{< /highlight >}} @@ -173,11 +173,11 @@ shows the correct format for data types used when reading from and writing to BigQuery: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryDataTypes >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryDataTypes >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_data_types >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_data_types >}} {{< /highlight >}} @@ -276,11 +276,11 @@ The following code reads an entire table that contains weather station data and then extracts the `max_temperature` column. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadTable >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadTable >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_table >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_table >}} {{< /highlight >}} @@ -300,22 +300,22 @@ If you don't want to read an entire table, you can supply a query string to The following code uses a SQL query to only read the `max_temperature` column. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadQuery >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadQuery >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_query >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_query >}} {{< /highlight >}} You can also use BigQuery's standard SQL dialect with a query string, as shown in the following example: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadQueryStdSQL >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryReadQueryStdSQL >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_query_std_sql >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_query_std_sql >}} {{< /highlight >}} ### Using the BigQuery Storage API {#storage-api} @@ -546,11 +546,11 @@ The following example code shows how to create a `TableSchema` for a table with two fields (source and quote) of type string. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQuerySchemaObject >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQuerySchemaObject >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_schema_object >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_schema_object >}} {{< /highlight >}} @@ -591,11 +591,11 @@ The following example shows how to use a string to specify the same table schema as the previous example. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQuerySchemaJson >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQuerySchemaJson >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_schema >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_schema >}} {{< /highlight >}} @@ -680,11 +680,11 @@ collection. The following examples use this `PCollection` that contains quotes. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteInput >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteInput >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write_input >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write_input >}} {{< /highlight >}} @@ -704,11 +704,11 @@ creates a table if needed; if the table already exists, it will be replaced. {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteTable >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteTable >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write >}} {{< /highlight >}} @@ -723,7 +723,7 @@ be replaced. {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteFunction >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteFunction >}} {{< /highlight >}} {{< paragraph class="language-java" >}} @@ -774,7 +774,7 @@ different table for each year. {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteDynamicDestinations >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryWriteDynamicDestinations >}} {{< /highlight >}} {{< highlight py >}} @@ -810,7 +810,7 @@ This example generates one partition per day. {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTimePartitioning >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryTimePartitioning >}} {{< /highlight >}} {{< highlight py >}} diff --git a/website/www/site/content/en/documentation/io/developing-io-python.md b/website/www/site/content/en/documentation/io/developing-io-python.md index 3e581e01dd13..47b3ed075714 100644 --- a/website/www/site/content/en/documentation/io/developing-io-python.md +++ b/website/www/site/content/en/documentation/io/developing-io-python.md @@ -183,13 +183,13 @@ See [AvroSource](https://github.com/apache/beam/blob/master/sdks/python/apache_b The following example, `CountingSource`, demonstrates an implementation of `BoundedSource` and uses the SDK-provided `RangeTracker` called `OffsetRangeTracker`. {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_new_source >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_new_source >}} {{< /highlight >}} To read data from the source in your pipeline, use the `Read` transform: {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_use_new_source >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_use_new_source >}} {{< /highlight >}} **Note:** When you create a source that end-users are going to use, we @@ -262,23 +262,23 @@ to `_CountingSource`. Then, create the wrapper `PTransform`, called `ReadFromCountingSource`: {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_new_ptransform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_new_ptransform >}} {{< /highlight >}} Finally, read from the source: {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_use_ptransform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_source_use_ptransform >}} {{< /highlight >}} For the sink, rename `SimpleKVSink` to `_SimpleKVSink`. Then, create the wrapper `PTransform`, called `WriteToKVSink`: {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_sink_new_ptransform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_sink_new_ptransform >}} {{< /highlight >}} Finally, write to the sink: {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_sink_use_ptransform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_custom_sink_use_ptransform >}} {{< /highlight >}} diff --git a/website/www/site/content/en/documentation/patterns/ai-platform.md b/website/www/site/content/en/documentation/patterns/ai-platform.md new file mode 100644 index 000000000000..905f89584447 --- /dev/null +++ b/website/www/site/content/en/documentation/patterns/ai-platform.md @@ -0,0 +1,216 @@ +--- +title: "AI Platform integration patterns" +--- + + +# AI Platform integration patterns + +This page describes common patterns in pipelines with Google Cloud AI Platform transforms. + +{{< language-switcher java py >}} + +## Analysing the structure and meaning of text + +This section shows how to use [Google Cloud Natural Language API](https://cloud.google.com/natural-language) to perform text analysis. + +Beam provides a PTransform called [AnnotateText](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.gcp.naturallanguageml.html#apache_beam.ml.gcp.naturallanguageml.AnnotateText). The transform takes a PCollection of type [Document](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.gcp.naturallanguageml.html#apache_beam.ml.gcp.naturallanguageml.Document). Each Document object contains various information about text. This includes the content, whether it is a plain text or HTML, an optional language hint and other settings. +`AnnotateText` produces response object of type `AnnotateTextResponse` returned from the API. `AnnotateTextResponse` is a protobuf message which contains a lot of attributes, some of which are complex structures. + +Here is an example of a pipeline that creates in-memory PCollection of strings, changes each string to Document object and invokes Natural Language API. Then, for each response object, a function is called to extract certain results of analysis. + +{{< highlight py >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" nlp_analyze_text >}} +{{< /highlight >}} + +{{< highlight java >}} +// Java examples will be available on Beam 2.23 release. +{{< /highlight >}} + + +### Extracting sentiments + +This is a part of response object returned from the API. Sentence-level sentiments can be found in `sentences` attribute. `sentences` behaves like a standard Python sequence, therefore all core language features (like iteration or slicing) will work. Overall sentiment can be found in `document_sentiment` attribute. + +``` +sentences { + text { + content: "My experience so far has been fantastic!" + } + sentiment { + magnitude: 0.8999999761581421 + score: 0.8999999761581421 + } +} +sentences { + text { + content: "I\'d really recommend this product." + begin_offset: 41 + } + sentiment { + magnitude: 0.8999999761581421 + score: 0.8999999761581421 + } +} + +...many lines omitted + +document_sentiment { + magnitude: 1.899999976158142 + score: 0.8999999761581421 +} +``` + +The function for extracting information about sentence-level and document-level sentiments is shown in the next code snippet. + +{{< highlight py >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" nlp_extract_sentiments >}} +{{< /highlight >}} + +{{< highlight java >}} +// Java examples will be available on Beam 2.23 release. +{{< /highlight >}} + +The snippet loops over `sentences` and, for each sentence, extracts the sentiment score. + +The output is: + +``` +{"sentences": [{"My experience so far has been fantastic!": 0.8999999761581421}, {"I'd really recommend this product.": 0.8999999761581421}], "document_sentiment": 0.8999999761581421} +``` + +### Extracting entities + +The next function inspects the response for entities and returns the names and the types of those entities. + +{{< highlight py >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" nlp_extract_entities >}} +{{< /highlight >}} + +{{< highlight java >}} +// Java examples will be available on Beam 2.23 release. +{{< /highlight >}} + +Entities can be found in `entities` attribute. Just like before, `entities` is a sequence, that's why list comprehension is a viable choice. The most tricky part is interpreting the types of entities. Natural Language API defines entity types as enum. In a response object, entity types are returned as integers. That's why a user has to instantiate `naturallanguageml.enums.Entity.Type` to access a human-readable name. + +The output is: + +``` +[{"name": "experience", "type": "OTHER"}, {"name": "product", "type": "CONSUMER_GOOD"}] +``` + +### Accessing sentence dependency tree + +The following code loops over the sentences and, for each sentence, builds an adjacency list that represents a dependency tree. For more information on what dependency tree is, see [Morphology & Dependency Trees](https://cloud.google.com/natural-language/docs/morphology#dependency_trees). + +{{< highlight py >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" analyze_dependency_tree >}} +{{< /highlight >}} + +{{< highlight java >}} +// Java examples will be available on Beam 2.23 release. +{{< /highlight >}} + +The output is below. For better readability, indexes are replaced by text which they refer to: + +``` +[ + { + "experience": [ + "My" + ], + "been": [ + "experience", + "far", + "has", + "been", + "fantastic", + "!" + ], + "far": [ + "so" + ] + }, + { + "recommend": [ + "I", + "'d", + "really", + "recommend", + "product", + "." + ], + "product": [ + "this" + ] + } +] +``` + +## Getting predictions + +This section shows how to use [Google Cloud AI Platform Prediction](https://cloud.google.com/ai-platform/prediction/docs/overview) to make predictions about new data from a cloud-hosted machine learning model. + +[tfx_bsl](https://github.com/tensorflow/tfx-bsl) is a library with a Beam PTransform called `RunInference`. `RunInference` is able to perform an inference that can use an external service endpoint for receiving data. When using a service endpoint, the transform takes a PCollection of type `tf.train.Example` and, for every batch of elements, sends a request to AI Platform Prediction. The size of a batch is automatically computed. For more details on how Beam finds the best batch size, refer to a docstring for [BatchElements](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.util.html?highlight=batchelements#apache_beam.transforms.util.BatchElements). Currently, the transform does not support using `tf.train.SequenceExample` as input, but the work is in progress. + + The transform produces a PCollection of type `PredictionLog`, which contains predictions. + +Before getting started, deploy a TensorFlow model to AI Platform Prediction. The cloud service manages the infrastructure needed to handle prediction requests in both efficient and scalable way. Do note that only TensorFlow models are supported by the transform. For more information, see [Exporting a SavedModel for prediction](https://cloud.google.com/ai-platform/prediction/docs/exporting-savedmodel-for-prediction). + +Once a machine learning model is deployed, prepare a list of instances to get predictions for. To send binary data, make sure that the name of an input ends in `_bytes`. This will base64-encode data before sending a request. + +### Example +Here is an example of a pipeline that reads input instances from the file, converts JSON objects to `tf.train.Example` objects and sends data to AI Platform Prediction. The content of a file can look like this: + +``` +{"input": "the quick brown"} +{"input": "la bruja le"} +``` + +The example creates `tf.train.BytesList` instances, thus it expects byte-like strings as input. However, other data types, like `tf.train.FloatList` and `tf.train.Int64List`, are also supported by the transform. + +Here is the code: + +{{< highlight py >}} +import json + +import apache_beam as beam + +import tensorflow as tf +from tfx_bsl.beam.run_inference import RunInference +from tfx_bsl.proto import model_spec_pb2 + +def convert_json_to_tf_example(json_obj): + samples = json.loads(json_obj) + for name, text in samples.items(): + value = tf.train.Feature(bytes_list=tf.train.BytesList( + value=[text.encode('utf-8')])) + feature = {name: value} + return tf.train.Example(features=tf.train.Features(feature=feature)) + +with beam.Pipeline() as p: + _ = (p + | beam.io.ReadFromText('gs://my-bucket/samples.json') + | beam.Map(convert_json_to_tf_example) + | RunInference( + model_spec_pb2.InferenceEndpoint( + model_endpoint_spec=model_spec_pb2.AIPlatformPredictionModelSpec( + project_id='my-project-id', + model_name='my-model-name', + version_name='my-model-version')))) +{{< /highlight >}} + +{{< highlight java >}} +// Getting predictions is not yet available for Java. [BEAM-9501] +{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/patterns/bigqueryio.md b/website/www/site/content/en/documentation/patterns/bigqueryio.md new file mode 100644 index 000000000000..1206c5deace6 --- /dev/null +++ b/website/www/site/content/en/documentation/patterns/bigqueryio.md @@ -0,0 +1,46 @@ +--- +layout: section +title: "BigQuery patterns" +section_menu: section-menu/documentation.html +permalink: /documentation/patterns/bigqueryio/ +--- + + +# Google BigQuery patterns + +The samples on this page show you common patterns for use with BigQueryIO. + +{{< language-switcher java py >}} + +## BigQueryIO deadletter pattern +In production systems, it is useful to implement the deadletter pattern with BigQueryIO outputting any elements which had errors during processing by BigQueryIO into another PCollection for further processing. +The samples below print the errors, but in a production system they can be sent to a deadletter table for later correction. + +{{< paragraph class="language-java" >}} +When using `STREAMING_INSERTS` you can use the `WriteResult` object to access a `PCollection` with the `TableRows` that failed to be inserted into BigQuery. +If you also set the `withExtendedErrorInfo` property , you will be able to access a `PCollection` from the `WriteResult`. The `PCollection` will then include a reference to the table, the data row and the `InsertErrors`. Which errors are added to the deadletter queue is determined via the `InsertRetryPolicy`. +{{< /paragraph >}} + +{{< paragraph class="language-py" >}} +In the result tuple you can access `FailedRows` to access the failed inserts. +{{< /paragraph >}} + +{{< highlight java >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" BigQueryIODeadLetter >}} +{{< /highlight >}} + +{{< highlight py >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" BigQueryIODeadLetter >}} +{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/patterns/custom-windows.md b/website/www/site/content/en/documentation/patterns/custom-windows.md index 22b5b285435d..23e83c46ada8 100644 --- a/website/www/site/content/en/documentation/patterns/custom-windows.md +++ b/website/www/site/content/en/documentation/patterns/custom-windows.md @@ -27,7 +27,7 @@ You can modify the [`assignWindows`](https://beam.apache.org/releases/javadoc/cu Access the `assignWindows` function through `WindowFn.AssignContext.element()`. The original, fixed-duration `assignWindows` function is: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow1 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow1 >}} {{< /highlight >}} ### Creating data-driven gaps @@ -38,13 +38,13 @@ To create data-driven gaps, add the following snippets to the `assignWindows` fu For example, the following function assigns each element to a window between the timestamp and `gapDuration`: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow3 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow3 >}} {{< /highlight >}} Then, set the `gapDuration` field in a windowing function: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow2 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow2 >}} {{< /highlight >}} ### Windowing messages into sessions @@ -53,13 +53,13 @@ After creating data-driven gaps, you can window incoming data into the new, cust First, set the session length to the gap duration: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow4 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow4 >}} {{< /highlight >}} Lastly, window data into sessions in your pipeline: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow6 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CustomSessionWindow6 >}} {{< /highlight >}} ### Example data and windows @@ -103,4 +103,4 @@ user=user-1, score=9, window=[2019-05-26T14:30:33.276Z..2019-05-26T14:30:41.849Z user=user-2, score=10, window=[2019-05-26T14:30:37.357Z..2019-05-26T14:30:47.357Z) ``` -With dynamic sessions, User #2 gets different scores. The third messages arrives seven seconds after the second message, so it's grouped into a different session. The large, 18-point session is split into two 9-point sessions. \ No newline at end of file +With dynamic sessions, User #2 gets different scores. The third messages arrives seven seconds after the second message, so it's grouped into a different session. The large, 18-point session is split into two 9-point sessions. diff --git a/website/www/site/content/en/documentation/patterns/file-processing.md b/website/www/site/content/en/documentation/patterns/file-processing.md index f119cd34da0a..d5d53dca558c 100644 --- a/website/www/site/content/en/documentation/patterns/file-processing.md +++ b/website/www/site/content/en/documentation/patterns/file-processing.md @@ -36,7 +36,7 @@ Use the [`FileIO`](https://beam.apache.org/releases/javadoc/current/org/apache/b {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternProcessNewFilesSnip1 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternProcessNewFilesSnip1 >}} {{< /highlight >}} {{< paragraph class="language-java" >}} @@ -44,7 +44,7 @@ The [`TextIO`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/ {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternProcessNewFilesSnip2 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternProcessNewFilesSnip2 >}} {{< /highlight >}} {{< paragraph class="language-java" >}} @@ -98,9 +98,9 @@ To read filenames in a pipeline job: {{< /paragraph >}} {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternAccessMetadataSnip1 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" FileProcessPatternAccessMetadataSnip1 >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" FileProcessPatternAccessMetadataSnip1 >}} -{{< /highlight >}} \ No newline at end of file +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" FileProcessPatternAccessMetadataSnip1 >}} +{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/patterns/overview.md b/website/www/site/content/en/documentation/patterns/overview.md index a610c3c83243..3bc682d2ab97 100644 --- a/website/www/site/content/en/documentation/patterns/overview.md +++ b/website/www/site/content/en/documentation/patterns/overview.md @@ -35,6 +35,13 @@ Pipeline patterns demonstrate common Beam use cases. Pipeline patterns are based **Custom window patterns** - Patterns for windowing functions * [Using data to dynamically set session window gaps](/documentation/patterns/custom-windows/#using-data-to-dynamically-set-session-window-gaps) +**BigQuery patterns** - Patterns for BigQueryIO +* [Google BigQuery patterns](/documentation/patterns/bigqueryio/#google-bigquery-patterns) + +**AI Platform integration patterns** - Patterns for using Google Cloud AI Platform transforms +* [Analysing the structure and meaning of text](/documentation/patterns/ai-platform/#analysing-the-structure-and-meaning-of-text) +* [Getting predictions](/documentation/patterns/ai-platform/#getting-predictions) + ## Contributing a pattern To contribute a new pipeline pattern, create an issue with the [`pipeline-patterns` label](https://issues.apache.org/jira/browse/BEAM-7449?jql=labels%20%3D%20pipeline-patterns) and add details to the issue description. See [Get started contributing](/contribute/) for more information. diff --git a/website/www/site/content/en/documentation/patterns/pipeline-options.md b/website/www/site/content/en/documentation/patterns/pipeline-options.md index 5bd379ea02f2..6cd1a7d14712 100644 --- a/website/www/site/content/en/documentation/patterns/pipeline-options.md +++ b/website/www/site/content/en/documentation/patterns/pipeline-options.md @@ -28,9 +28,9 @@ Use the `ValueProvider` interface to access runtime parameters after completing You can use the `ValueProvider` interface to pass runtime parameters to your pipeline, but you can only log the parameters from within the the Beam DAG. A solution is to add a pipeline [branch](/documentation/programming-guide/#applying-transforms) with a `DoFn` that processes a placeholder value and then logs the runtime parameters: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" AccessingValueProviderInfoAfterRunSnip1 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" AccessingValueProviderInfoAfterRunSnip1 >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" AccessingValueProviderInfoAfterRunSnip1 >}} -{{< /highlight >}} \ No newline at end of file +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" AccessingValueProviderInfoAfterRunSnip1 >}} +{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/patterns/side-inputs.md b/website/www/site/content/en/documentation/patterns/side-inputs.md index 06cd2e6b247d..5d60acc20568 100644 --- a/website/www/site/content/en/documentation/patterns/side-inputs.md +++ b/website/www/site/content/en/documentation/patterns/side-inputs.md @@ -42,7 +42,7 @@ The global window side input triggers on processing time, so the main pipeline n For instance, the following code sample uses a `Map` to create a `DoFn`. The `Map` becomes a `View.asSingleton` side input that’s rebuilt on each counter tick. The side input updates every 5 seconds in order to demonstrate the workflow. In a real-world scenario, the side input would typically update every few hours or once per day. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" SideInputPatternSlowUpdateGlobalWindowSnip1 >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" SideInputPatternSlowUpdateGlobalWindowSnip1 >}} {{< /highlight >}} {{< highlight py >}} @@ -70,9 +70,9 @@ PCollection element. 1. Apply the side input. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" PeriodicallyUpdatingSideInputs >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" PeriodicallyUpdatingSideInputs >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" SideInputSlowUpdateSnip1 >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" SideInputSlowUpdateSnip1 >}} {{< /highlight >}} diff --git a/website/www/site/content/en/documentation/programming-guide.md b/website/www/site/content/en/documentation/programming-guide.md index c30578a0ec86..a9f5f0962341 100644 --- a/website/www/site/content/en/documentation/programming-guide.md +++ b/website/www/site/content/en/documentation/programming-guide.md @@ -122,7 +122,7 @@ Pipeline p = Pipeline.create(options); {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_creating >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_creating >}} {{< /highlight >}} {{< highlight go >}} @@ -157,7 +157,7 @@ PipelineOptions options = {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_creating >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_creating >}} {{< /highlight >}} {{< highlight go >}} @@ -199,7 +199,7 @@ public interface MyOptions extends PipelineOptions { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_options_define_custom >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_options_define_custom >}} {{< /highlight >}} {{< highlight go >}} @@ -229,7 +229,7 @@ public interface MyOptions extends PipelineOptions { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_options_define_custom_with_help_and_default >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_options_define_custom_with_help_and_default >}} {{< /highlight >}} {{< highlight go >}} @@ -316,7 +316,7 @@ public static void main(String[] args) { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_reading >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipelines_constructing_reading >}} {{< /highlight >}} {{< highlight go >}} @@ -369,7 +369,7 @@ public static void main(String[] args) { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_pcollection >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_pcollection >}} {{< /highlight >}} ### 3.2. PCollection characteristics {#pcollection-characteristics} @@ -631,8 +631,8 @@ PCollection wordLengths = words.apply( words = ... # The DoFn to perform on each element in the input PCollection. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_pardo >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_apply >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_pardo >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_apply >}} {{< /highlight >}} {{< highlight go >}} @@ -708,7 +708,7 @@ static class ComputeWordLengthFn extends DoFn { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_pardo >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_pardo >}} {{< /highlight >}} {{< paragraph class="language-java" >}} @@ -771,7 +771,7 @@ words = ... # Apply a lambda function to the PCollection words. # Save the result as the PCollection word_lengths. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_using_flatmap >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_using_flatmap >}} {{< /highlight >}} {{< highlight go >}} @@ -810,7 +810,7 @@ words = ... # Apply a Map with a lambda function to the PCollection words. # Save the result as the PCollection word_lengths. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_using_map >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_using_map >}} {{< /highlight >}} {{< paragraph class="language-java" >}} @@ -966,22 +966,22 @@ data contains names and phone numbers. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleInputs >}} +{{< code_sample "examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleInputs >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_inputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_inputs >}} {{< /highlight >}} After `CoGroupByKey`, the resulting data contains all data associated with each unique key from any of the input collections. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleOutputs >}} +{{< code_sample "examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleOutputs >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_outputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_outputs >}} {{< /highlight >}} The following code example joins the two `PCollection`s with `CoGroupByKey`, @@ -989,21 +989,21 @@ followed by a `ParDo` to consume the result. Then, the code uses tags to look up and format data from each collection. {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CoGroupByKeyTuple >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java" CoGroupByKeyTuple >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_group_by_key_cogroupbykey_tuple >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_group_by_key_cogroupbykey_tuple >}} {{< /highlight >}} The formatted data looks like this: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleFormattedOutputs >}} +{{< code_sample "examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java" CoGroupByKeyTupleFormattedOutputs >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_formatted_outputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_group_by_key_cogroupbykey_tuple_formatted_outputs >}} {{< /highlight >}} #### 4.2.4. Combine {#combine} @@ -1048,7 +1048,7 @@ public static class SumInts implements SerializableFunction, I {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_bounded_sum >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_bounded_sum >}} {{< /highlight >}} ##### 4.2.4.2. Advanced combinations using CombineFn {#advanced-combines} @@ -1121,7 +1121,7 @@ public class AverageFn extends CombineFn { {{< highlight py >}} pc = ... -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_custom_average_define >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_custom_average_define >}} {{< /highlight >}} ##### 4.2.4.3. Combining a PCollection into a single value {#combining-pcollection} @@ -1145,7 +1145,7 @@ PCollection sum = pc.apply( # The resulting PCollection, called result, contains one value: the sum of all # the elements in the input PCollection. pc = ... -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_custom_average_execute >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_custom_average_execute >}} {{< /highlight >}} ##### 4.2.4.4. Combine and global windowing {#combine-global-windowing} @@ -1229,7 +1229,7 @@ PCollection> avgAccuracyPerPlayer = # PCollection is grouped by key and the numeric values associated with each key # are averaged into a float. player_accuracies = ... -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_per_key >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" combine_per_key >}} {{< /highlight >}} #### 4.2.5. Flatten {#flatten} @@ -1258,7 +1258,7 @@ PCollection merged = collections.apply(Flatten.pCollections()); {{< highlight py >}} # Flatten takes a tuple of PCollection objects. # Returns a single PCollection that contains all of the elements in the PCollection objects in that tuple. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_flatten >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_flatten >}} {{< /highlight >}} ##### 4.2.5.1. Data encoding in merged collections {#data-encoding-merged-collections} @@ -1321,10 +1321,10 @@ PCollection fortiethPercentile = studentsByPercentile.get(4); # Provide an int value with the desired number of result partitions, and a partitioning function (partition_fn in this example). # Returns a tuple of PCollection objects containing each of the resulting partitions as individual PCollection objects. students = ... -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_partition >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_partition >}} # You can extract each partition from the tuple of PCollection objects as follows: -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_partition_40th >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_multiple_pcollections_partition_40th >}} {{< /highlight >}} ### 4.3. Requirements for writing user code for Beam transforms {#requirements-for-writing-user-code-for-beam-transforms} @@ -1448,12 +1448,12 @@ determined by the input data, or depend on a different branch of your pipeline. # of the actual elements of pcoll being passed into each process invocation. In this example, side inputs are # passed to a FlatMap transform as extra arguments and consumed by filter_using_length. words = ... -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_side_input >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_side_input >}} # We can also pass side inputs to a ParDo transform, which will get passed to its process method. # The first two arguments for the process method would be self and element. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_side_input_dofn >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_side_input_dofn >}} ... {{< /highlight >}} @@ -1550,12 +1550,12 @@ together. # with_outputs are attributes on the returned DoOutputsTuple object. The tags give access to the # corresponding output PCollections. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_tagged_outputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_tagged_outputs >}} # The result is also iterable, ordered in the same order that the tags were passed to with_outputs(), # the main tag (if specified) first. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_tagged_outputs_iter >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_tagged_outputs_iter >}} {{< /highlight >}} #### 4.5.2. Emitting to multiple outputs in your DoFn {#multiple-outputs-dofn} @@ -1588,12 +1588,12 @@ together. # using the pvalue.OutputValue wrapper class. # Based on the previous example, this shows the DoFn emitting to the main output and two additional outputs. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_emitting_values_on_tagged_outputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_emitting_values_on_tagged_outputs >}} # Producing multiple outputs is also available in Map and FlatMap. # Here is an example that uses FlatMap and shows that the tags do not need to be specified ahead of time. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_undeclared_outputs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_pardo_with_undeclared_outputs >}} {{< /highlight >}} #### 4.5.3. Accessing additional parameters in your DoFn {#other-dofn-parameters} @@ -1839,7 +1839,7 @@ transform's intermediate data changes type multiple times. {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_monitoring_composite >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" pipeline_monitoring_composite >}} {{< /highlight >}} #### 4.6.2. Creating a composite transform {#composite-transform-creation} @@ -1867,7 +1867,7 @@ The following code sample shows how to declare a `PTransform` that accepts a {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_transform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_transform >}} {{< /highlight >}} Within your `PTransform` subclass, you'll need to override the `expand` method. @@ -1891,7 +1891,7 @@ The following code sample shows how to override `expand` for the {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_transform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_transform >}} {{< /highlight >}} As long as you override the `expand` method in your `PTransform` subclass to @@ -1969,7 +1969,7 @@ p.apply("ReadFromText", {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_pipelineio_read >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_pipelineio_read >}} {{< /highlight >}} To read data from disparate sources into a single `PCollection`, read each one @@ -1994,7 +1994,7 @@ records.apply("WriteToText", {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" model_pipelineio_write >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_pipelineio_write >}} {{< /highlight >}} ### 5.4. Beam-provided I/O transforms {#provided-io-transforms} @@ -2020,7 +2020,7 @@ Most structured records share some common characteristics: languages: int, long, string, etc. * Often a field type can be marked as optional (sometimes referred to as nullable) or required. -Oten records have a nested structure. A nested structure occurs when a field itself has subfields so the +Often records have a nested structure. A nested structure occurs when a field itself has subfields so the type of the field itself has a schema. Fields that are array or map types is also a common feature of these structured records. @@ -2362,7 +2362,7 @@ This logical type allows creating an enumeration type consisting of a set of nam {{< highlight java >}} Schema schema = Schema.builder() … - .addLogicalTypeField(“color”, EnumerationType.create(“RED”, “GREEN”, “BLUE”)) + .addLogicalTypeField("color", EnumerationType.create("RED", "GREEN", "BLUE")) .build(); {{< /highlight >}} @@ -2370,15 +2370,15 @@ The value of this field is stored in the row as an INT32 type, however the logic you access the enumeration either as a string or a value. For example: {{< highlight java >}} -EnumerationType.Value enumValue = enumType.valueOf(“RED”); +EnumerationType.Value enumValue = enumType.valueOf("RED"); enumValue.getValue(); // Returns 0, the integer value of the constant. -enumValue.toString(); // Returns “RED”, the string value of the constant +enumValue.toString(); // Returns "RED", the string value of the constant {{< /highlight >}} Given a row object with an enumeration field, you can also extract the field as the enumeration value. {{< highlight java >}} -EnumerationType.Value enumValue = row.getLogicalTypeValue(“color”, EnumerationType.Value.class); +EnumerationType.Value enumValue = row.getLogicalTypeValue("color", EnumerationType.Value.class); {{< /highlight >}} Automatic schema inference from Java POJOs and JavaBeans automatically converts Java enums to EnumerationType logical @@ -2391,10 +2391,10 @@ OneOfType allows creating a disjoint union type over a set of schema fields. For {{< highlight java >}} Schema schema = Schema.builder() … - .addLogicalTypeField(“oneOfField”, - OneOfType.create(Field.of(“intField”, FieldType.INT32), - Field.of(“stringField”, FieldType.STRING), - Field.of(“bytesField”, FieldType.BYTES))) + .addLogicalTypeField("oneOfField", + OneOfType.create(Field.of("intField", FieldType.INT32), + Field.of("stringField", FieldType.STRING), + Field.of("bytesField", FieldType.BYTES))) .build(); {{< /highlight >}} @@ -2405,19 +2405,19 @@ logical type however defines a Value object that contains an enumeration value i {{< highlight java >}} // Returns an enumeration indicating all possible case values for the enum. // For the above example, this will be -// EnumerationType.create(“intField”, “stringField”, “bytesField”); +// EnumerationType.create("intField", "stringField", "bytesField"); EnumerationType oneOfEnum = onOfType.getCaseEnumType(); // Creates an instance of the union with the string field set. -OneOfType.Value oneOfValue = oneOfType.createValue(“stringField”, “foobar”); +OneOfType.Value oneOfValue = oneOfType.createValue("stringField", "foobar"); // Handle the oneof switch (oneOfValue.getCaseEnumType().toString()) { - case “intField”: + case "intField": return processInt(oneOfValue.getValue(Integer.class)); - case “stringField”: + case "stringField": return processString(oneOfValue.getValue(String.class)); - case “bytesField”: + case "bytesField": return processBytes(oneOfValue.getValue(bytes[].class)); } {{< /highlight >}} @@ -2565,7 +2565,7 @@ In order to select a field at the top level of a schema, the name of the field i the user ids from a `PCollection` of purchases one would write (using the `Select` transform) {{< highlight java >}} -purchases.apply(Select.fieldNames(“userId”)); +purchases.apply(Select.fieldNames("userId")); {{< /highlight >}} ##### **Nested fields** @@ -2574,7 +2574,7 @@ Individual nested fields can be specified using the dot operator. For example, t shipping address one would write {{< highlight java >}} -purchases.apply(Select.fieldNames(“shippingAddress.postCode”)); +purchases.apply(Select.fieldNames("shippingAddress.postCode")); {{< /highlight >}} ##### **Wildcards** @@ -2583,7 +2583,7 @@ The * operator can be specified at any nesting level to represent all fields at shipping-address fields one would write {{< highlight java >}} -purchases.apply(Select.fieldNames(“shippingAddress.*”)); +purchases.apply(Select.fieldNames("shippingAddress.*")); {{< /highlight >}} ##### **Arrays** @@ -2592,7 +2592,7 @@ An array field, where the array element type is a row, can also have subfields o selected, the result is an array of the selected subfield type. For example {{< highlight java >}} -purchases.apply(Select.fieldNames(“transactions[].bank”)); +purchases.apply(Select.fieldNames("transactions[].bank")); {{< /highlight >}} Will result in a row containing an array field with element-type string, containing the list of banks for each @@ -2631,7 +2631,7 @@ specific keys from the map. For example, given the following schema: The following {{< highlight java >}} -purchasesByType.apply(Select.fieldNames(“purchases{}.userId”)); +purchasesByType.apply(Select.fieldNames("purchases{}.userId")); {{< /highlight >}} Will result in a row containing an map field with key-type string and value-type string. The selected map will contain @@ -2655,7 +2655,7 @@ field as a top-level field. Both top-level and nested fields can be selected. Fo could select only the userId and streetAddress fields as follows {{< highlight java >}} -purchases.apply(Select.fieldNames(“userId”, shippingAddress.streetAddress”)); +purchases.apply(Select.fieldNames("userId", "shippingAddress.streetAddress")); {{< /highlight >}} The resulting `PCollection` will have the following schema @@ -2683,7 +2683,7 @@ The resulting `PCollection` will have the following schema The same is true for wildcard selections. The following {{< highlight java >}} -purchases.apply(Select.fieldNames(“userId”, shippingAddress.*”)); +purchases.apply(Select.fieldNames("userId", "shippingAddress.*")); {{< /highlight >}} Will result in the following schema @@ -2729,7 +2729,7 @@ top-level field in the resulting row. This means that if multiple fields are sel selected field will appear as its own array field. For example {{< highlight java >}} -purchases.apply(Select.fieldNames( “transactions.bank”, transactions.purchaseAmount”)); +purchases.apply(Select.fieldNames( "transactions.bank", "transactions.purchaseAmount")); {{< /highlight >}} Will result in the following schema @@ -2832,7 +2832,7 @@ The simplest usage of `Group` specifies no aggregations, in which case all input are grouped together into an `ITERABLE` field. For example {{< highlight java >}} -purchases.apply(Group.byFieldNames(“userId”, shippingAddress.streetAddress”)); +purchases.apply(Group.byFieldNames("userId", "shippingAddress.streetAddress")); {{< /highlight >}} The output schema of this is: @@ -2863,9 +2863,9 @@ The names of the key and values fields in the output schema can be controlled us builders, as follows: {{< highlight java >}} -purchases.apply(Group.byFieldNames(“userId”, shippingAddress.streetAddress”) - .withKeyField(“userAndStreet”) - .withValueField(“matchingPurchases”)); +purchases.apply(Group.byFieldNames("userId", "shippingAddress.streetAddress") + .withKeyField("userAndStreet") + .withValueField("matchingPurchases")); {{< /highlight >}} It is quite common to apply one or more aggregations to the grouped result. Each aggregation can specify one or more fields @@ -2874,10 +2874,10 @@ following application computes three aggregations grouped by userId, with all ag output schema: {{< highlight java >}} -purchases.apply(Group.byFieldNames(“userId”) - .aggregateField(“itemId”, Count.combineFn(), “numPurchases”) - .aggregateField(“costCents”, Sum.ofLongs(), “totalSpendCents”) - .aggregateField(“costCents”, Top.largestLongsFn(10), “topPurchases”)); +purchases.apply(Group.byFieldNames("userId") + .aggregateField("itemId", Count.combineFn(), "numPurchases") + .aggregateField("costCents", Sum.ofLongs(), "totalSpendCents") + .aggregateField("costCents", Top.largestLongsFn(10), "topPurchases")); {{< /highlight >}} The result of this aggregation will have the following schema: @@ -2915,7 +2915,7 @@ and is specified with the `using` keyword: PCollection transactions = readTransactions(); PCollection reviews = readReviews(); PCollection joined = transactions.apply( - Join.innerJoin(reviews).using(“userId”, “productId”)); + Join.innerJoin(reviews).using("userId", "productId")); {{< /highlight >}} The resulting schema is the following: @@ -2948,8 +2948,8 @@ Review schema named those fields differently than the Transaction schema, then w PCollection joined = transactions.apply( Join.innerJoin(reviews).on( FieldsEqual - .left(“userId”, “productId”) - .right(“reviewUserId”, “reviewProductId”))); + .left("userId", "productId") + .right("reviewUserId", "reviewProductId"))); {{< /highlight >}} In addition to inner joins, the Join transform supports full outer joins, left outer joins, and right outer joins. @@ -2971,8 +2971,8 @@ which all predicates return true will pass the filter. For example the following {{< highlight java >}} purchases.apply(Filter - .whereFieldName(“costCents”, c -> c > 100 * 20) - .whereFieldName(“shippingAddress.country”, c -> c.equals(“de”)); + .whereFieldName("costCents", c -> c > 100 * 20) + .whereFieldName("shippingAddress.country", c -> c.equals("de")); {{< /highlight >}} Will produce all purchases made from Germany with a purchase price of greater than twenty cents. @@ -2989,9 +2989,9 @@ For example, the following application {{< highlight java >}} purchases.apply(AddFields.create() - .field(“timeOfDaySeconds”, FieldType.INT32) - .field(“shippingAddress.deliveryNotes”, FieldType.STRING) - .field(“transactions.isFlagged, FieldType.BOOLEAN, false)); + .field("timeOfDaySeconds", FieldType.INT32) + .field("shippingAddress.deliveryNotes", FieldType.STRING) + .field("transactions.isFlagged, FieldType.BOOLEAN, false)); {{< /highlight >}} Results in a `PCollection` with an expanded schema. All of the rows and fields of the input, but also with the specified @@ -3007,7 +3007,7 @@ syntax. For example, the following snippet {{< highlight java >}} -purchases.apply(DropFields.fields(“userId”, “shippingAddress.streetAddress”)); +purchases.apply(DropFields.fields("userId", "shippingAddress.streetAddress")); {{< /highlight >}} Results in a copy of the input with those two fields and their corresponding values removed. @@ -3024,8 +3024,8 @@ For example, the following snippet {{< highlight java >}} purchases.apply(RenameFields.create() - .rename(“userId”, “userIdentifier”) - .rename(“shippingAddress.streetAddress”, “shippingAddress.street”)); + .rename("userId", "userIdentifier") + .rename("shippingAddress.streetAddress", "shippingAddress.street")); {{< /highlight >}} Results in the same set of unmodified input elements, however the schema on the PCollection has been changed to rename @@ -3113,7 +3113,7 @@ using the above-described selection expressions, as follows: {{< highlight java >}} purchases.appy(ParDo.of(new DoFn() { @ProcessElement public void process( - @FieldAccess(“userId”) String userId, @FieldAccess(“itemId”) long itemId) { + @FieldAccess("userId") String userId, @FieldAccess("itemId") long itemId) { ... } })); @@ -3124,7 +3124,7 @@ You can also select nested fields, as follows. {{< highlight java >}} purchases.appy(ParDo.of(new DoFn() { @ProcessElement public void process( - @FieldAccess(“shippingAddress.street”) String street) { + @FieldAccess("shippingAddress.street") String street) { ... } })); @@ -3660,7 +3660,7 @@ into fixed windows, each 60 seconds in length: {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_fixed_windows >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_fixed_windows >}} {{< /highlight >}} #### 8.3.2. Sliding time windows {#using-sliding-time-windows} @@ -3676,7 +3676,7 @@ begins every five seconds: {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_sliding_windows >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_sliding_windows >}} {{< /highlight >}} #### 8.3.3. Session windows {#using-session-windows} @@ -3692,7 +3692,7 @@ least 10 minutes (600 seconds): {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_session_windows >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_session_windows >}} {{< /highlight >}} Note that the sessions are per-key — each key in the collection will have its @@ -3711,7 +3711,7 @@ a single global window for a `PCollection`: {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_global_window >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_global_window >}} {{< /highlight >}} ### 8.4. Watermarks and late data {#watermarks-and-late-data} @@ -3822,7 +3822,7 @@ with a `DoFn` to attach the timestamps to each element in your `PCollection`. {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_timestamp >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" setting_timestamp >}} {{< /highlight >}} ## 9. Triggers {#triggers} @@ -3911,7 +3911,7 @@ firings: {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_early_late_triggers >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_early_late_triggers >}} {{< /highlight >}} #### 9.1.1. Default trigger {#default-trigger} @@ -3953,8 +3953,8 @@ It is important to note that if, for example, you specify AfterCount(50) and only 32 elements arrive, those 32 elements sit around forever. If the 32 elements are important to you, consider using [composite triggers](#composite-triggers) to combine multiple -conditions. This allows you to specify multiple firing conditions such as “fire -either when I receive 50 elements, or every 1 second”. +conditions. This allows you to specify multiple firing conditions such as "fire +either when I receive 50 elements, or every 1 second". ### 9.4. Setting a trigger {#setting-a-trigger} @@ -3988,7 +3988,7 @@ sets the window's **accumulation mode**. {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_setting_trigger >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_setting_trigger >}} {{< /highlight >}} #### 9.4.1. Window accumulation modes {#window-accumulation-modes} @@ -4155,7 +4155,7 @@ example trigger code fires on the following conditions: {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_triggers >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_composite_triggers >}} {{< /highlight >}} #### 9.5.3. Other composite triggers {#other-composite-triggers} @@ -4171,7 +4171,7 @@ elements, or after a minute. {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" model_other_composite_triggers >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" model_other_composite_triggers >}} {{< /highlight >}} ## 10. Metrics {#metrics} @@ -4876,4 +4876,4 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { isTimerSetState.clear(); } })); -{{< /highlight >}} \ No newline at end of file +{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/resources/videos-and-podcasts.md b/website/www/site/content/en/documentation/resources/videos-and-podcasts.md index b1d0a0bb6371..651cd8856cd8 100644 --- a/website/www/site/content/en/documentation/resources/videos-and-podcasts.md +++ b/website/www/site/content/en/documentation/resources/videos-and-podcasts.md @@ -65,6 +65,24 @@ Presented by Frances Perry, *Apache Beam PPMC member* The following resources present Apache Beam partnerships. +### Distributed Processing for Machine Learning Production Pipelines + +Flink Forward, 2020 + +Presented by Ahmet Altay, Robert Crowe, Reza Rokni + + +
+ +### TensorFlow Extended: An End-to-End Machine Learning Platform for TensorFlow + +Spark+AI, San Francisco, 2019 + +Presented by Konstantinos Katsiapis, Ahmet Altay + + +
+ ### Flink and Beam: Current State & Roadmap Flink Forward, Berlin, 2016 @@ -129,4 +147,4 @@ Your browser does not support the audio element. ## Next Steps -* Take a self-paced tour through our [Learning Resources](/documentation/resources/learning-resources). \ No newline at end of file +* Take a self-paced tour through our [Learning Resources](/documentation/resources/learning-resources). diff --git a/website/www/site/content/en/documentation/sdks/python-type-safety.md b/website/www/site/content/en/documentation/sdks/python-type-safety.md index 5147664ba94a..4b966db084c4 100644 --- a/website/www/site/content/en/documentation/sdks/python-type-safety.md +++ b/website/www/site/content/en/documentation/sdks/python-type-safety.md @@ -45,13 +45,13 @@ Introducing type hints for the `PTransforms` you define allows you to catch pote Consider the following example, in which `numbers` is a `PCollection` of `str` values: {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_missing_define_numbers >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_missing_define_numbers >}} {{< /highlight >}} The code then applies a `Filter` transform to the `numbers` collection with a callable that retrieves the even numbers. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_missing_apply >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_missing_apply >}} {{< /highlight >}} When you call `p.run()`, this code generates an error when trying to execute this transform because `Filter` expects a `PCollection` of integers, but is given a `PCollection` of strings instead. @@ -87,7 +87,7 @@ Annotations are currently supported on: The following code declares an `int` input and a `str` output type hint on the `to_id` transform, using annotations on `my_fn`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_map_annotations >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_map_annotations >}} {{< /highlight >}} The following code declares `int` input and output type hints on `filter_evens`, using annotations on `FilterEvensDoFn.process`. @@ -97,7 +97,7 @@ It is an error to have a non-iterable return type annotation for these functions Other supported iterable types include: `Iterator`, `Generator`, `Tuple`, `List`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_do_fn_annotations >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_do_fn_annotations >}} {{< /highlight >}} The following code declares `int` input and output type hints on `double_evens`, using annotations on `FilterEvensDoubleDoFn.process`. @@ -105,7 +105,7 @@ Since `process` returns a `list` or `None`, the output type is annotated as `Opt Beam will also remove the outer `Optional` and (as above) the outer iterable of the return type, only on the `DoFn.process` method and functions passed to `FlatMap`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_do_fn_annotations_optional >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test_py3.py" type_hints_do_fn_annotations_optional >}} {{< /highlight >}} ### Declaring Type Hints Inline @@ -113,7 +113,7 @@ Beam will also remove the outer `Optional` and (as above) the outer iterable of To specify type hints inline, use the methods `with_input_types` and `with_output_types`. The following example code declares an input type hint inline: {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_takes >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_takes >}} {{< /highlight >}} When you apply the Filter transform to the numbers collection in the example above, you'll be able to catch the error during pipeline construction. @@ -125,7 +125,7 @@ To specify type hints as properties of a `DoFn` or `PTransform`, use the decorat The following code declares an `int` type hint on `FilterEvensDoFn`, using the decorator `@with_input_types()`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_do_fn >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_do_fn >}} {{< /highlight >}} Decorators receive an arbitrary number of positional and/or keyword arguments, typically interpreted in the context of the function they're wrapping. Generally the first argument is a type hint for the main input, and additional arguments are type hints for side inputs. @@ -147,7 +147,7 @@ The following code specifies an input type hint that asserts the generic type `T If the input to `MyTransform` is of type `str`, Beam will infer the output type to be `Tuple[int, str]`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_transform >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_transform >}} {{< /highlight >}} ## Kinds of Type Hints @@ -200,13 +200,13 @@ In addition to using type hints for type checking at pipeline construction, you For example, the following pipeline emits elements of the wrong type. Depending on the runner implementation, its execution may or may not fail at runtime. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_runtime_off >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_runtime_off >}} {{< /highlight >}} However, if you enable runtime type checking, the code is guaranteed to fail at runtime. To enable runtime type checking, set the pipeline option `runtime_type_check` to `True`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_runtime_on >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_runtime_on >}} {{< /highlight >}} Note that because runtime type checks are done for each `PCollection` element, enabling this feature may incur a significant performance penalty. It is therefore recommended that runtime type checks are disabled for production pipelines. @@ -230,5 +230,5 @@ For example, suppose you have a `PCollection` of key-value pairs whose keys are The following code shows the example `Player` class and how to define a `Coder` for it. When you use type hints, Beam infers which `Coders` to use, using `beam.coders.registry`. The following code registers `PlayerCoder` as a coder for the `Player` class. In the example, the input type declared for `CombinePerKey` is `Tuple[Player, int]`. In this case, Beam infers that the `Coder` objects to use are `TupleCoder`, `PlayerCoder`, and `IntCoder`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_deterministic_key >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets_test.py" type_hints_deterministic_key >}} {{< /highlight >}} diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md b/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md index 5d29149ba285..6a157f169452 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md @@ -37,7 +37,7 @@ Then, we apply `Filter` in multiple ways to filter out produce by their duration We define a function `is_perennial` which returns `True` if the element's duration equals `'perennial'`, and `False` otherwise. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_function >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_function >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -45,7 +45,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} {{< buttons-code-snippet @@ -57,7 +57,7 @@ Output `PCollection` after `Filter`: We can also use lambda functions to simplify **Example 1**. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_lambda >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_lambda >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -65,7 +65,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} {{< buttons-code-snippet @@ -80,7 +80,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `has_duration` takes `plant` and `duration` as arguments. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_multiple_arguments >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_multiple_arguments >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -88,7 +88,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} {{< buttons-code-snippet @@ -104,7 +104,7 @@ In this example, we pass a `PCollection` the value `'perennial'` as a singleton. We then use that value to filter out perennials. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_singleton >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_singleton >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -112,7 +112,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} {{< buttons-code-snippet @@ -126,7 +126,7 @@ This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_iter >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_iter >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -134,7 +134,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" valid_plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" valid_plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -152,7 +152,7 @@ Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_dict >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_dict >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -160,7 +160,7 @@ Output `PCollection` after `Filter`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md b/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md index 5fbdad6d88cf..872805b38637 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md @@ -38,7 +38,7 @@ We use the function `str.split` which takes a single `str` element and outputs a This pipeline splits the input element using whitespaces, creating a list of zero or more elements. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_simple >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_simple >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -46,7 +46,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -58,7 +58,7 @@ Output `PCollection` after `FlatMap`: We define a function `split_words` which splits an input `str` element using the delimiter `','` and outputs a `list` of `str`s. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_function >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_function >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -66,7 +66,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} @@ -81,7 +81,7 @@ Each input element is already an `iterable`, where each element is what we want We use a lambda function that returns the same input element it received. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_lambda >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_lambda >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -89,7 +89,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -103,7 +103,7 @@ We use a generator to iterate over the input list and yield each of the elements Each yielded result in the generator is an element in the resulting `PCollection`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_generator >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_generator >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -111,7 +111,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -124,7 +124,7 @@ If your `PCollection` consists of `(key, value)` pairs, you can use `FlatMapTuple` to unpack them into different function arguments. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_tuple >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_tuple >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -132,7 +132,7 @@ Output `PCollection` after `FlatMapTuple`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -147,7 +147,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `split_words` takes `text` and `delimiter` as arguments. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_multiple_arguments >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_multiple_arguments >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -155,7 +155,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -171,7 +171,7 @@ In this example, we pass a `PCollection` the value `','` as a singleton. We then use that value as the delimiter for the `str.split` method. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_singleton >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_singleton >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -179,7 +179,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -193,7 +193,7 @@ This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_iter >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_iter >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -201,7 +201,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -219,7 +219,7 @@ Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_dict >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_dict >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -227,7 +227,7 @@ Output `PCollection` after `FlatMap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md b/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md index 013e36e7a504..65b4cba33266 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md @@ -29,7 +29,7 @@ In the following example, we create a pipeline with a `PCollection` of key-value Then, we apply `Keys` to extract the keys and discard the values. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys.py" keys >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys.py" keys >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -37,7 +37,7 @@ Output `PCollection` after `Keys`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys_test.py" icons >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys_test.py" icons >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md b/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md index 298cbaaf37ea..0bfe13df1baa 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md @@ -30,7 +30,7 @@ In the following example, we create a pipeline with a `PCollection` of key-value Then, we apply `KvSwap` to swap the keys and values. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap.py" kvswap >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap.py" kvswap >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -38,7 +38,7 @@ Output `PCollection` after `KvSwap`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/map.md b/website/www/site/content/en/documentation/transforms/python/elementwise/map.md index 0a0c31ff6e69..f3036fea0c5b 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/map.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/map.md @@ -36,7 +36,7 @@ We use the function `str.strip` which takes a single `str` element and outputs a It strips the input element's whitespaces, including newlines and tabs. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_simple >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_simple >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -44,7 +44,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -56,7 +56,7 @@ Output `PCollection` after `Map`: We define a function `strip_header_and_newline` which strips any `'#'`, `' '`, and `'\n'` characters from each element. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_function >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_function >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -64,7 +64,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -76,7 +76,7 @@ Output `PCollection` after `Map`: We can also use lambda functions to simplify **Example 2**. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_lambda >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_lambda >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -84,7 +84,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -99,7 +99,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `strip` takes `text` and `chars` as arguments. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_multiple_arguments >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_multiple_arguments >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -107,7 +107,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -120,7 +120,7 @@ If your `PCollection` consists of `(key, value)` pairs, you can use `MapTuple` to unpack them into different function arguments. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_tuple >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_tuple >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -128,7 +128,7 @@ Output `PCollection` after `MapTuple`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -144,7 +144,7 @@ In this example, we pass a `PCollection` the value `'# \n'` as a singleton. We then use that value as the characters for the `str.strip` method. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_singleton >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_singleton >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -152,7 +152,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -166,7 +166,7 @@ This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_iter >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_iter >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -174,7 +174,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -192,7 +192,7 @@ Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_dict >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_dict >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -200,7 +200,7 @@ Output `PCollection` after `Map`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plant_details >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plant_details >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md b/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md index c0514693954a..cd10cdeae67a 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md @@ -42,7 +42,7 @@ The `process` method is called once per element, and it can yield zero or more output elements. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn >}} {{}} {{< paragraph class="notebook-skip" >}} @@ -50,7 +50,7 @@ Output `PCollection` after `ParDo`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -71,7 +71,7 @@ In this example, we add new parameters to the `process` method to bind parameter object. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_params >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_params >}} {{}} {{< paragraph class="notebook-skip" >}} @@ -79,7 +79,7 @@ In this example, we add new parameters to the `process` method to bind parameter {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" dofn_params >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" dofn_params >}} {{< /highlight >}} {{< buttons-code-snippet @@ -132,7 +132,7 @@ starts and finishes with `start_bundle` and `finish_bundle`. For example, if the worker crashes, `teardown` might not be called. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_methods >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_methods >}} {{}} {{< paragraph class="notebook-skip" >}} @@ -140,7 +140,7 @@ starts and finishes with `start_bundle` and `finish_bundle`. {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" results >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" results >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md b/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md index 5034086cd763..72f6aa0502bd 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md @@ -47,7 +47,7 @@ In the following example, we have a known list of durations. We partition the `PCollection` into one `PCollection` for every duration type. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_function >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_function >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -55,7 +55,7 @@ Output `PCollection`s: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} {{< /highlight >}} {{< buttons-code-snippet @@ -67,7 +67,7 @@ Output `PCollection`s: We can also use lambda functions to simplify **Example 1**. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_lambda >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_lambda >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -75,7 +75,7 @@ Output `PCollection`s: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} {{< /highlight >}} {{< buttons-code-snippet @@ -115,7 +115,7 @@ This `split_dataset` function is generic enough to support any number of partiti You might want to adapt the bucket assignment to use a more appropriate or randomized hash for your dataset. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_multiple_arguments >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_multiple_arguments >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -123,7 +123,7 @@ Output `PCollection`s: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" train_test >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" train_test >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md b/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md index 3374f4c25d48..2e55c1589257 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md @@ -62,7 +62,7 @@ To start matching at any point instead of the beginning of the string, use [`Regex.find(regex)`](#example-4-regex-find). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -70,7 +70,7 @@ Output `PCollection` after `Regex.matches`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} {{< /highlight >}} {{< buttons-code-snippet @@ -91,7 +91,7 @@ To start matching at any point instead of the beginning of the string, use [`Regex.find_all(regex, group=Regex.ALL, outputEmpty=False)`](#example-5-regex-find-all). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_all_matches >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_all_matches >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -99,7 +99,7 @@ Output `PCollection` after `Regex.all_matches`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_all_matches >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_all_matches >}} {{< /highlight >}} {{< buttons-code-snippet @@ -121,7 +121,7 @@ To start matching at any point instead of the beginning of the string, use [`Regex.find_kv(regex, keyGroup)`](#example-6-regex-find-as-key-value-pairs). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches_kv >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches_kv >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -129,7 +129,7 @@ Output `PCollection` after `Regex.matches_kv`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches_kv >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches_kv >}} {{< /highlight >}} {{< buttons-code-snippet @@ -151,7 +151,7 @@ If you need to match from the start only, consider using [`Regex.matches(regex)`](#example-1-regex-match). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -159,7 +159,7 @@ Output `PCollection` after `Regex.find`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} {{< /highlight >}} {{< buttons-code-snippet @@ -181,7 +181,7 @@ If you need to match all groups from the start only, consider using [`Regex.all_matches(regex)`](#example-2-regex-match-with-all-groups). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_all >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_all >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -189,7 +189,7 @@ Output `PCollection` after `Regex.find_all`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_all >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_all >}} {{< /highlight >}} {{< buttons-code-snippet @@ -212,7 +212,7 @@ If you need to match as key-value pairs from the start only, consider using [`Regex.matches_kv(regex)`](#example-3-regex-match-into-key-value-pairs). {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_kv >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_kv >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -220,7 +220,7 @@ Output `PCollection` after `Regex.find_kv`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_kv >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_kv >}} {{< /highlight >}} {{< buttons-code-snippet @@ -235,7 +235,7 @@ You can also use on the `replacement`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_all >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_all >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -243,7 +243,7 @@ Output `PCollection` after `Regex.replace_all`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_all >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_all >}} {{< /highlight >}} {{< buttons-code-snippet @@ -258,7 +258,7 @@ You can also use on the `replacement`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_first >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_first >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -266,7 +266,7 @@ Output `PCollection` after `Regex.replace_first`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_first >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_first >}} {{< /highlight >}} {{< buttons-code-snippet @@ -279,7 +279,7 @@ Output `PCollection` after `Regex.replace_first`: The argument `outputEmpty` is set to `False` by default, but can be set to `True` to keep empty items in the output list. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_split >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_split >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -287,7 +287,7 @@ Output `PCollection` after `Regex.split`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_split >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_split >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md b/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md index 292825c41ba7..61a2070fd388 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md @@ -36,7 +36,7 @@ The following example converts a `(key, value)` pair into a string delimited by You can specify a different delimiter using the `delimiter` argument. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_kvs >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_kvs >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -44,7 +44,7 @@ Output `PCollection` after `ToString`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet @@ -57,7 +57,7 @@ The following example converts a dictionary into a string. The string output will be equivalent to `str(element)`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_element >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_element >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -65,7 +65,7 @@ Output `PCollection` after `ToString`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plant_lists >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plant_lists >}} {{< /highlight >}} {{< buttons-code-snippet @@ -80,7 +80,7 @@ You can specify a different delimiter using the `delimiter` argument. The string output will be equivalent to `iterable.join(delimiter)`. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_iterables >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_iterables >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -88,7 +88,7 @@ Output `PCollection` after `ToString`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants_csv >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants_csv >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/values.md b/website/www/site/content/en/documentation/transforms/python/elementwise/values.md index b1179596da93..11e54a4fdbc7 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/values.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/values.md @@ -29,7 +29,7 @@ In the following example, we create a pipeline with a `PCollection` of key-value Then, we apply `Values` to extract the values and discard the keys. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/values.py" values >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/values.py" values >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -37,7 +37,7 @@ Output `PCollection` after `Values`: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/values_test.py" plants >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/values_test.py" plants >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md b/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md index 3a1c6fd9a10b..3cc037217b3f 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md @@ -34,7 +34,7 @@ The elements themselves often already contain a timestamp field. in the form of seconds. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_event_time >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_event_time >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -42,7 +42,7 @@ Output `PCollection` after getting the timestamps: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_timestamps >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_timestamps >}} {{< /highlight >}} {{< buttons-code-snippet @@ -57,7 +57,7 @@ For more information on time formatting options, see [`time.strftime`](https://docs.python.org/3/library/time.html#time.strftime). {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" time_tuple2unix_time >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" time_tuple2unix_time >}} {{< /highlight >}} To convert from a @@ -66,7 +66,7 @@ to `unix_time` you can use convert it to a `time.struct_time` first with [`datetime.timetuple`](https://docs.python.org/3/library/datetime.html#datetime.datetime.timetuple). {{< highlight >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" datetime2unix_time >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" datetime2unix_time >}} {{< /highlight >}} ### Example 2: Timestamp by logical clock @@ -76,7 +76,7 @@ If each element has a chronological number, these numbers can be used as a These numbers have to be converted to a *"seconds"* equivalent, which can be especially important depending on your windowing and late data rules. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_logical_clock >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_logical_clock >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -84,7 +84,7 @@ Output `PCollection` after getting the timestamps: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_events >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_events >}} {{< /highlight >}} {{< buttons-code-snippet @@ -100,7 +100,7 @@ Workers might have time deltas, so using this method is not a reliable way to do By using processing time, there is no way of knowing if data is arriving late because the timestamp is attached when the element *enters* into the pipeline. {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_processing_time >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_processing_time >}} {{< /highlight >}} {{< paragraph class="notebook-skip" >}} @@ -108,7 +108,7 @@ Output `PCollection` after getting the timestamps: {{< /paragraph >}} {{< highlight class="notebook-skip" >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_processing_times >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_processing_times >}} {{< /highlight >}} {{< buttons-code-snippet diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index 76a032c89369..a4aaa0bfff83 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -87,10 +87,17 @@ versions denoted `0.x.y`. ## Releases +### 2.21.0 (2020-05-27) +Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.21.0/apache-beam-2.21.0-source-release.zip). +[SHA-512](https://downloads.apache.org/beam/2.21.0/apache-beam-2.21.0-source-release.zip.sha512). +[signature](https://downloads.apache.org/beam/2.21.0/apache-beam-2.21.0-source-release.zip.asc). + +[Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347143). + ### 2.20.0 (2020-04-15) -Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.20.0/apache-beam-2.20.0-source-release.zip). -[SHA-512](https://downloads.apache.org/beam/2.20.0/apache-beam-2.20.0-source-release.zip.sha512). -[signature](https://downloads.apache.org/beam/2.20.0/apache-beam-2.20.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.20.0/apache-beam-2.20.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.20.0/apache-beam-2.20.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.20.0/apache-beam-2.20.0-source-release.zip.asc). [Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12346780). @@ -241,4 +248,3 @@ Official [source code download](https://archive.apache.org/dist/beam/2.0.0/apach [signature](https://archive.apache.org/dist/beam/2.0.0/apache-beam-2.0.0-source-release.zip.asc). [Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12339746). - diff --git a/website/www/site/content/en/get-started/mobile-gaming-example.md b/website/www/site/content/en/get-started/mobile-gaming-example.md index 9054f5373e10..6a976c56f42f 100644 --- a/website/www/site/content/en/get-started/mobile-gaming-example.md +++ b/website/www/site/content/en/get-started/mobile-gaming-example.md @@ -101,11 +101,11 @@ This example uses batch processing, and the diagram's Y axis represents processi After reading the score events from the input file, the pipeline groups all of those user/score pairs together and sums the score values into one total value per unique user. `UserScore` encapsulates the core logic for that step as the [user-defined composite transform](/documentation/programming-guide/#composite-transforms) `ExtractAndSumScore`: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java" DocInclude_USExtractXform >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java" DocInclude_USExtractXform >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/user_score.py" extract_and_sum_score >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/user_score.py" extract_and_sum_score >}} {{< /highlight >}} `ExtractAndSumScore` is written to be more general, in that you can pass in the field by which you want to group the data (in the case of our game, by unique user or unique team). This means we can re-use `ExtractAndSumScore` in other pipelines that group score data by team, for example. @@ -113,11 +113,11 @@ After reading the score events from the input file, the pipeline groups all of t Here's the main method of `UserScore`, showing how we apply all three steps of the pipeline: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java" DocInclude_USMain >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java" DocInclude_USMain >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/user_score.py" main >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/user_score.py" main >}} {{< /highlight >}} ### Limitations @@ -186,11 +186,11 @@ Beam's windowing feature uses the [intrinsic timestamp information](/documentati The following code shows this: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSAddTsAndWindow >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSAddTsAndWindow >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" add_timestamp_and_window >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" add_timestamp_and_window >}} {{< /highlight >}} Notice that the transforms the pipeline uses to specify the windowing are distinct from the actual data processing transforms (such as `ExtractAndSumScores`). This functionality provides you some flexibility in designing your Beam pipeline, in that you can run existing transforms over datasets with different windowing characteristics. @@ -206,11 +206,11 @@ It also lets the pipeline include relevant **late data**—data events with vali The following code shows how `HourlyTeamScore` uses the `Filter` transform to filter events that occur either before or after the relevant analysis period: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSFilters >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSFilters >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" filter_by_time_range >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" filter_by_time_range >}} {{< /highlight >}} #### Calculating Score Per Team, Per Window @@ -218,11 +218,11 @@ The following code shows how `HourlyTeamScore` uses the `Filter` transform to fi `HourlyTeamScore` uses the same `ExtractAndSumScores` transform as the `UserScore` pipeline, but passes a different key (team, as opposed to user). Also, because the pipeline applies `ExtractAndSumScores` _after_ applying fixed-time 1-hour windowing to the input data, the data gets grouped by both team _and_ window. You can see the full sequence of transforms in `HourlyTeamScore`'s main method: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSMain >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java" DocInclude_HTSMain >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" main >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/hourly_team_score.py" main >}} {{< /highlight >}} ### Limitations @@ -279,11 +279,11 @@ As processing time advances and more scores are processed, the trigger outputs t The following code example shows how `LeaderBoard` sets the processing time trigger to output the data for user scores: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java" DocInclude_ProcTimeTrigger >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java" DocInclude_ProcTimeTrigger >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/leader_board.py" processing_time_trigger >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/leader_board.py" processing_time_trigger >}} {{< /highlight >}} `LeaderBoard` sets the [window accumulation mode](/documentation/programming-guide/#window-accumulation-modes) to accumulate window panes as the trigger fires. This accumulation mode is set by invoking `.accumulatingFiredPanes` using `accumulation_mode=trigger.AccumulationMode.ACCUMULATING` when setting the trigger, and causes the pipeline to accumulate the previously emitted data together with any new data that's arrived since the last trigger fire. This ensures that `LeaderBoard` is a running sum for the user scores, rather than a collection of individual sums. @@ -313,11 +313,11 @@ Data arriving above the solid watermark line is _late data_ — this is a score The following code example shows how `LeaderBoard` applies fixed-time windowing with the appropriate triggers to have our pipeline perform the calculations we want: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java" DocInclude_WindowAndTrigger >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java" DocInclude_WindowAndTrigger >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/leader_board.py" window_and_trigger >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/leader_board.py" window_and_trigger >}} {{< /highlight >}} Taken together, these processing strategies let us address the latency and completeness issues present in the `UserScore` and `HourlyTeamScore` pipelines, while still using the same basic transforms to process the data—as a matter of fact, both calculations still use the same `ExtractAndSumScore` transform that we used in both the `UserScore` and `HourlyTeamScore` pipelines. @@ -356,21 +356,21 @@ Since the average depends on the pipeline data, we need to calculate it, and the The following code example shows the composite transform that handles abuse detection. The transform uses the `Sum.integersPerKey` transform to sum all scores per user, and then the `Mean.globally` transform to determine the average score for all users. Once that's been calculated (as a `PCollectionView` singleton), we can pass it to the filtering `ParDo` using `.withSideInputs`: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_AbuseDetect >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_AbuseDetect >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/game_stats.py" abuse_detect >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/game_stats.py" abuse_detect >}} {{< /highlight >}} The abuse-detection transform generates a view of users supected to be spambots. Later in the pipeline, we use that view to filter out any such users when we calculate the team score per hour, again by using the side input mechanism. The following code example shows where we insert the spam filter, between windowing the scores into fixed windows and extracting the team scores: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_FilterAndCalc >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_FilterAndCalc >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/game_stats.py" filter_and_calc >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/game_stats.py" filter_and_calc >}} {{< /highlight >}} #### Analyzing Usage Patterns @@ -390,21 +390,21 @@ between instances are.* We can use the session-windowed data to determine the average length of uninterrupted play time for all of our users, as well as the total score they achieve during each session. We can do this in the code by first applying session windows, summing the score per user and session, and then using a transform to calculate the length of each individual session: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_SessionCalc >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_SessionCalc >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/game_stats.py" session_calc >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/game_stats.py" session_calc >}} {{< /highlight >}} This gives us a set of user sessions, each with an attached duration. We can then calculate the _average_ session length by re-windowing the data into fixed time windows, and then calculating the average for all sessions that end in each hour: {{< highlight java >}} -{{< github_sample "/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_Rewindow >}} +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/complete/game/GameStats.java" DocInclude_Rewindow >}} {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/complete/game/game_stats.py" rewindow >}} +{{< code_sample "sdks/python/apache_beam/examples/complete/game/game_stats.py" rewindow >}} {{< /highlight >}} We can use the resulting information to find, for example, what times of day our users are playing the longest, or which stretches of the day are more likely to see shorter play sessions. diff --git a/website/www/site/content/en/get-started/wordcount-example.md b/website/www/site/content/en/get-started/wordcount-example.md index 58ae28507e34..08e447912fc6 100644 --- a/website/www/site/content/en/get-started/wordcount-example.md +++ b/website/www/site/content/en/get-started/wordcount-example.md @@ -123,7 +123,7 @@ sections, we will specify the pipeline's runner. {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_options >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_options >}} {{< /highlight >}} {{< paragraph class="language-java language-py" >}} @@ -143,7 +143,7 @@ Pipeline p = Pipeline.create(options); {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_create >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_create >}} {{< /highlight >}} {{< highlight go >}} @@ -179,7 +179,7 @@ p.apply(TextIO.read().from("gs://apache-beam-samples/shakespeare/*")) {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_read >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_read >}} {{< /highlight >}} {{< highlight go >}} @@ -204,7 +204,7 @@ lines := textio.Read(s, "gs://apache-beam-samples/shakespeare/*") {{< highlight py >}} # The Flatmap transform is a simplified version of ParDo. -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_pardo >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_pardo >}} {{< /highlight >}} {{< highlight go >}} @@ -231,7 +231,7 @@ words := beam.ParDo(s, func(line string, emit func(string)) { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_count >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_count >}} {{< /highlight >}} {{< highlight go >}} @@ -253,7 +253,7 @@ counted := stats.Count(s, words) {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_map >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_map >}} {{< /highlight >}} {{< highlight go >}} @@ -272,7 +272,7 @@ formatted := beam.ParDo(s, func(w string, c int) string { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_write >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_write >}} {{< /highlight >}} {{< highlight go >}} @@ -304,7 +304,7 @@ p.run().waitUntilFinish(); {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_run >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_minimal_run >}} {{< /highlight >}} {{< highlight go >}} @@ -534,7 +534,7 @@ static class ExtractWordsFn extends DoFn { {{< highlight py >}} # In this example, the DoFns are defined as classes: -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_dofn >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_dofn >}} {{< /highlight >}} {{< highlight go >}} @@ -603,7 +603,7 @@ public static void main(String[] args) throws IOException { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_composite >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_composite >}} {{< /highlight >}} {{< highlight go >}} @@ -652,7 +652,7 @@ public static void main(String[] args) { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_options >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" examples_wordcount_wordcount_options >}} {{< /highlight >}} {{< highlight go >}} @@ -862,7 +862,7 @@ public class DebuggingWordCount { {{< /highlight >}} {{< highlight py >}} -{{< github_sample "/apache/beam/blob/master/sdks/python/apache_beam/examples/snippets/snippets.py" example_wordcount_debugging_logging >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" example_wordcount_debugging_logging >}} {{< /highlight >}} {{< highlight go >}} diff --git a/website/www/site/content/en/roadmap/connectors-multi-sdk.md b/website/www/site/content/en/roadmap/connectors-multi-sdk.md index e00d6e424fe1..500c44c58545 100644 --- a/website/www/site/content/en/roadmap/connectors-multi-sdk.md +++ b/website/www/site/content/en/roadmap/connectors-multi-sdk.md @@ -25,7 +25,7 @@ efforts. See [Beam portability framework roadmap](https://beam.apache.org/roadma # Cross-language transforms -_Last updated on November 2019._ +_Last updated on May 2020._ As an added benefit of Beam portability effort, we are able to utilize Beam transforms across SDKs. This has many benefits. @@ -33,7 +33,7 @@ As an added benefit of Beam portability effort, we are able to utilize Beam tran + Beam pipelines written using Python and Go SDKs will be able to utilize the vast selection of connectors that are currently implemented for Java SDK. + Java SDK will be able to utilize connectors for systems that only offer a Python API. + Go SDK, will be able to utilize connectors currently available for Java and Python SDKs. -* Ease of developing and maintaining Beam transforms - in general, with cross-language transforms, Beam transform authors will be able to implement new Beam transforms using a +* Ease of developing and maintaining Beam transforms - in general, with cross-language transforms, Beam transform authors will be able to implement new Beam transforms using a language of choice and utilize these transforms from other languages reducing the maintenance and support overheads. * [Beam SQL](https://beam.apache.org/documentation/dsls/sql/overview/), that is currently only available to Java SDK, will become available to Python and Go SDKs. * [Beam TFX transforms](https://www.tensorflow.org/tfx/transform/get_started), that are currently only available to Beam Python SDK pipelines will become available to Java and Go SDKs. @@ -48,9 +48,9 @@ Work related to developing/updating the cross-language transforms API for Java/P * Basic API for Java SDK - completed * Basic API for Python SDK - completed -* Basic API for Go SDK - Not started +* Basic API for Go SDK - In progress * Basic cross-language transform expansion service for Java and Python SDKs - completed -* Artifact staging - In progress - [email thread](https://lists.apache.org/thread.html/6fcee7047f53cf1c0636fb65367ef70842016d57effe2e5795c4137d@%3Cdev.beam.apache.org%3E), [doc](https://docs.google.com/document/d/1XaiNekAY2sptuQRIXpjGAyaYdSc-wlJ-VKjl04c8N48/edit#heading=h.900gc947qrw8) +* Artifact staging - mostly completed - [email thread](https://lists.apache.org/thread.html/6fcee7047f53cf1c0636fb65367ef70842016d57effe2e5795c4137d@%3Cdev.beam.apache.org%3E), [doc](https://docs.google.com/document/d/1XaiNekAY2sptuQRIXpjGAyaYdSc-wlJ-VKjl04c8N48/edit#heading=h.900gc947qrw8) ### Support for Flink runner @@ -62,27 +62,32 @@ Work related to making cross-language transforms available for Flink runner. Work related to making cross-language transforms available for Dataflow runner. -* Basic support for executing cross-language transforms on Dataflow runner +* Basic support for executing cross-language transforms on Dataflow runner + This work requires updates to Dataflow service's job submission and job execution logic. This is currently being developed at Google. ### Support for Direct runner Work related to making cross-language transforms available on Direct runner -* Basic support for executing cross-language transforms on portable Direct runner - Not started +* Basic support for executing cross-language transforms on Pyton Direct runner - completed +* Basic support for executing cross-language transforms on Java Direct runner - Not started ### Connector/transform support Ongoing and planned work related to making existing connectors/transforms available to other SDKs through the cross-language transforms framework. -* Java KafkIO - In progress - [BEAM-7029](https://issues.apache.org/jira/browse/BEAM-7029) +* Java JdbcIO - In progress - [BEAM-10135](https://issues.apache.org/jira/browse/BEAM-10135), [BEAM-10136](https://issues.apache.org/jira/browse/BEAM-10136) +* Java KafkaIO - completed - [BEAM-7029](https://issues.apache.org/jira/browse/BEAM-7029) +* Java KinesisIO - In progress - [BEAM-10137](https://issues.apache.org/jira/browse/BEAM-10137), [BEAM-10138](https://issues.apache.org/jira/browse/BEAM-10138) * Java PubSubIO - In progress - [BEAM-7738](https://issues.apache.org/jira/browse/BEAM-7738) +* Java SpannerIO - In progress - [BEAM-10139](https://issues.apache.org/jira/browse/BEAM-10139), [BEAM-10140](https://issues.apache.org/jira/browse/BEAM-10140) +* Java SQL - completed - [BEAM-8603](https://issues.apache.org/jira/browse/BEAM-8603) ### Portable Beam schema Portable Beam schema support will provide a generalized mechanism for serializing and transferring data across language boundaries which will be extremely useful for pipelines that employ cross-language transforms. -* Make row coder a standard coder and implement in python - In progress - [BEAM-7886](https://issues.apache.org/jira/browse/BEAM-7886) +* Make row coder a standard coder and implement in python - completed - [BEAM-7886](https://issues.apache.org/jira/browse/BEAM-7886) ### Integration/Performance testing @@ -94,4 +99,3 @@ Work related to adding documenting on cross-language transforms to Beam Website. * Document cross-language transforms API for Java/Python - Not started * Document API for making existing transforms available as cross-language transforms for Java/Python - Not started - diff --git a/website/www/site/content/en/roadmap/go-sdk.md b/website/www/site/content/en/roadmap/go-sdk.md index 8e003e689cca..4172cb09cb59 100644 --- a/website/www/site/content/en/roadmap/go-sdk.md +++ b/website/www/site/content/en/roadmap/go-sdk.md @@ -18,7 +18,7 @@ limitations under the License. # Go SDK Roadmap The Go SDK is currently experimental. As the first purely portable Beam SDK, the Go SDK is constrained -by the status of the [Beam Portabillity Framework](https://beam.apache.org/roadmap/portability/) and the existence of +by the status of the [Beam Portability Framework](https://beam.apache.org/roadmap/portability/) and the existence of portable runners. **April 2020 Update** diff --git a/website/www/site/data/authors.yml b/website/www/site/data/authors.yml index 8a111a688cec..376cfdbaa62b 100644 --- a/website/www/site/data/authors.yml +++ b/website/www/site/data/authors.yml @@ -30,6 +30,9 @@ anton: ccy: name: Charles Chen email: ccy@apache.org +chadrik: + name: Chad Dombrova + email: chadrik@apache.org chamikara: name: Chamikara Jayalath email: chamikara@apache.org diff --git a/website/www/site/data/io_matrix.yaml b/website/www/site/data/io_matrix.yaml new file mode 100644 index 000000000000..3ce5a4f864ec --- /dev/null +++ b/website/www/site/data/io_matrix.yaml @@ -0,0 +1,377 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +categories: + - name: File-based + description: These I/O connectors involve working with files. + rows: + - transform: FileIO + description: "General-purpose transforms for working with files: listing files (matching), reading and writing." + implementations: + - language: java + name: org.apache.beam.sdk.io.FileIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileIO.html + - language: py + name: apache_beam.io.FileIO + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.fileio.html + - transform: AvroIO + description: PTransforms for reading from and writing to [Avro](https://avro.apache.org/) files. + implementations: + - language: java + name: org.apache.beam.sdk.io.AvroIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/AvroIO.html + - language: py + name: apache_beam.io.avroio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.avroio.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/avroio + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/avroio + - transform: TextIO + description: PTransforms for reading and writing text files. + implementations: + - language: java + name: org.apache.beam.sdk.io.TextIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TextIO.html + - language: py + name: apache_beam.io.textio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.textio.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/textio + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/textio + - transform: TFRecordIO + description: PTransforms for reading and writing [TensorFlow TFRecord](https://www.tensorflow.org/tutorials/load_data/tfrecord) files. + implementations: + - language: java + name: org.apache.beam.sdk.io.TFRecordIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TFRecordIO.html + - language: py + name: apache_beam.io.tfrecordio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.tfrecordio.html + - transform: XmlIO + description: Transforms for reading and writing XML files using [JAXB](https://www.oracle.com/technical-resources/articles/javase/jaxb.html) mappers. + implementations: + - language: java + name: org.apache.beam.sdk.io.xml.XmlIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/xml/XmlIO.html + - transform: TikaIO + description: Transforms for parsing arbitrary files using [Apache Tika](https://tika.apache.org/). + implementations: + - language: java + name: org.apache.beam.sdk.io.tika.TikaIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/tika/TikaIO.html + - transform: ParquetIO + description: IO for reading from and writing to [Parquet](https://parquet.apache.org/) files. + docs: /documentation/io/built-in/parquet/ + implementations: + - language: java + name: org.apache.beam.sdk.io.parquet.ParquetIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/parquet/ParquetIO.html + - language: py + name: apache_beam.io.parquetio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.parquetio.html + - transform: ThriftIO + description: PTransforms for reading and writing files containing [Thrift](https://thrift.apache.org/)-encoded data. + implementations: + - language: java + name: org.apache.beam.sdk.io.thrift.ThriftIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/thrift/ThriftIO.html + - transform: VcfIO + description: A source for reading from [VCF files](https://samtools.github.io/hts-specs/VCFv4.2.pdf) (version 4.x). + implementations: + - language: py + name: apache_beam.io.vcfio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.vcfio.html + - transform: S3IO + description: A source for reading from and writing to [Amazon S3](https://aws.amazon.com/s3/). + implementations: + - language: py + name: apache_beam.io.aws.s3io + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.aws.s3io.html + - transform: GcsIO + description: A source for reading from and writing to [Google Cloud Storage](https://cloud.google.com/storage). + implementations: + - language: py + name: apache_beam.io.gcp.gcsio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsio.html + - name: FileSystem + description: Beam provides a File system interface that defines APIs for writing file systems agnostic code. Several I/O connectors are implemented as a FileSystem implementation. + rows: + - transform: HadoopFileSystem + description: "`FileSystem` implementation for accessing [Hadoop](https://hadoop.apache.org/) Distributed File System files." + implementations: + - language: java + name: org.apache.beam.sdk.io.hdfs.HadoopFileSystemRegistrar + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.html + - language: py + name: apache_beam.io.hadoopfilesystem + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.hadoopfilesystem.html + - transform: GcsFileSystem + description: "`FileSystem` implementation for [Google Cloud Storage](https://cloud.google.com/storage)." + implementations: + - language: java + name: org.apache.beam.sdk.extensions.gcp.storage.GcsFileSystemRegistrar + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.html + - language: py + name: apache_beam.io.gcp.gcsfilesystem + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsfilesystem.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs + - transform: LocalFileSystem + description: "`FileSystem` implementation for accessing files on disk." + implementations: + - language: java + name: org.apache.beam.sdk.io.LocalFileSystemRegistrar + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/LocalFileSystemRegistrar.html + - language: py + name: apache_beam.io.localfilesystem + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.localfilesystem.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local + - transform: S3FileSystem + description: "`FileSystem` implementation for [Amazon S3](https://aws.amazon.com/s3/)." + implementations: + - language: java + name: org.apache.beam.sdk.io.aws.s3.S3FileSystemRegistrar + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/package-summary.html + - transform: In-memory + description: "`FileSystem` implementation in memory; useful for testing." + implementations: + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs + - name: Messaging + description: These I/O connectors typically involve working with unbounded sources that come from messaging sources. + rows: + - transform: KinesisIO + description: PTransforms for reading from and writing to [Kinesis](https://aws.amazon.com/kinesis/) streams. + implementations: + - language: java + name: org.apache.beam.sdk.io.kinesis.KinesisIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kinesis/KinesisIO.html + - transform: AmqpIO + description: AMQP 1.0 protocol using the Apache QPid Proton-J library + implementations: + - language: java + name: org.apache.beam.sdk.io.amqp.AmqpIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/amqp/AmqpIO.html + - transform: KafkaIO + description: Read and Write PTransforms for [Apache Kafka](https://kafka.apache.org/). + implementations: + - language: java + name: org.apache.beam.sdk.io.kafka.KafkaIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kafka/KafkaIO.html + - language: py + name: apache_beam.io.external.kafka + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.kafka.html + - transform: PubSubIO + description: Read and Write PTransforms for [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) streams. + implementations: + - language: java + name: org.apache.beam.sdk.io.gcp.pubsub.PubsubIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html + - language: py + name: apache_beam.io.gcp.pubsub + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html + - language: py + name: apache_beam.io.external.gcp.pubsub + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.gcp.pubsub.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio + - transform: JmsIO + description: An unbounded source for [JMS](https://www.oracle.com/java/technologies/java-message-service.html) destinations (queues or topics). + implementations: + - language: java + name: org.apache.beam.sdk.io.jms.JmsIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jms/JmsIO.html + - transform: MqttIO + description: An unbounded source for [MQTT](https://mqtt.org/) broker. + implementations: + - language: java + name: org.apache.beam.sdk.io.mqtt.MqttIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mqtt/MqttIO.html + - transform: RabbitMqIO + description: A IO to publish or consume messages with a RabbitMQ broker. + implementations: + - language: java + name: org.apache.beam.sdk.io.rabbitmq.RabbitMqIO + url: https://github.com/apache/beam/blob/master/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java + - transform: SqsIO + description: An unbounded source for [Amazon Simple Queue Service (SQS)](https://aws.amazon.com/sqs/). + implementations: + - language: java + name: org.apache.beam.sdk.io.aws.sqs.SqsIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sqs/SqsIO.html + - language: java + name: org.apache.beam.sdk.io.aws2.sqs.SqsIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sqs/SqsIO.html + - transform: SnsIO + description: PTransforms for writing to [Amazon Simple Notification Service (SNS)](https://aws.amazon.com/sns/). + implementations: + - language: java + name: org.apache.beam.sdk.io.aws.sns.SnsIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sns/SnsIO.html + - language: java + name: org.apache.beam.sdk.io.aws2.sns.SnsIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sns/SnsIO.html + - name: Database + description: These I/O connectors are used to connect to database systems. + rows: + - transform: CassandraIO + description: An IO to read from [Apache Cassandra](https://cassandra.apache.org/). + implementations: + - language: java + name: org.apache.beam.sdk.io.cassandra.CassandraIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/cassandra/CassandraIO.html + - transform: HadoopFormatIO + description: Allows for reading data from any source or writing data to any sink which implements [Hadoop](https://hadoop.apache.org/) InputFormat or OutputFormat. + docs: /documentation/io/built-in/hadoop/ + implementations: + - language: java + name: org.apache.beam.sdk.io.hadoop.format.HadoopFormatIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.html + - transform: HBaseIO + description: A bounded source and sink for [HBase](https://hbase.apache.org/). + implementations: + - language: java + name: org.apache.beam.sdk.io.hbase.HBaseIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hbase/HBaseIO.html + - transform: HCatalogIO + description: HCatalog source supports reading of HCatRecord from a [HCatalog](https://cwiki.apache.org/confluence/display/Hive/HCatalog)-managed source, for example [Hive](https://hive.apache.org/). + docs: /documentation/io/built-in/hcatalog/ + implementations: + - language: java + name: org.apache.beam.sdk.io.hcatalog.HCatalogIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hcatalog/HCatalogIO.html + - transform: KuduIO + description: A bounded source and sink for (Kudu)[https://kudu.apache.org/]. + implementations: + - language: java + name: org.apache.beam.sdk.io.kudu + url: https://github.com/apache/beam/blob/master/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java + - transform: SolrIO + description: Transforms for reading and writing data from/to [Solr](https://lucene.apache.org/solr/). + implementations: + - language: java + name: org.apache.beam.sdk.io.solr.SolrIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/solr/SolrIO.html + - transform: ElasticsearchIO + description: Transforms for reading and writing data from/to [Elasticsearch](https://www.elastic.co/elasticsearch/). + implementations: + - language: java + name: org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.html + - transform: BigQueryIO + description: Read from and write to [Google Cloud BigQuery](https://cloud.google.com/bigquery). + docs: /documentation/io/built-in/google-bigquery/ + implementations: + - language: java + name: org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.html + - language: py + name: apache_beam.io.gcp.bigquery + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigquery.html + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio + - transform: BigTableIO + description: Read from and write to [Google Cloud Bigtable](https://cloud.google.com/bigtable/). + implementations: + - language: java + name: org.apache.beam.sdk.io.gcp.bigtable.BigtableIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.html + - language: py + name: apache_beam.io.gcp.bigtableio module + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigtableio.html + - transform: DatastoreIO + description: Read from and write to [Google Cloud Datastore](https://cloud.google.com/datastore). + implementations: + - language: java + name: org.apache.beam.sdk.io.gcp.datastore.DatastoreIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/datastore/DatastoreIO.html + - language: py + name: apache_beam.io.gcp.datastore.v1new.datastoreio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.datastore.v1new.datastoreio.html + - transform: SpannerIO + description: Experimental Transforms for reading from and writing to [Google Cloud Spanner](https://cloud.google.com/spanner). + implementations: + - language: java + name: org.apache.beam.sdk.io.gcp.spanner.SpannerIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.html + - transform: JdbcIO + description: IO to read and write data on [JDBC](https://docs.oracle.com/javase/tutorial/jdbc/basics/index.html). + implementations: + - language: java + name: org.apache.beam.sdk.io.jdbc.JdbcIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jdbc/JdbcIO.html + - transform: MongoDbIO + description: IO to read and write data on [MongoDB](https://www.mongodb.com/). + implementations: + - language: java + name: org.apache.beam.sdk.io.mongodb.MongoDbIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbIO.html + - language: py + name: apache_beam.io.mongodbio + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.mongodbio.html + - transform: MongoDbGridFSIO + description: IO to read and write data on [MongoDB GridFS](https://docs.mongodb.com/manual/core/gridfs/). + implementations: + - language: java + name: org.apache.beam.sdk.io.mongodb.MongoDbGridFSIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.html + - transform: RedisIO + description: An IO to manipulate a [Redis](https://redis.io/) key/value database. + implementations: + - language: java + name: org.apache.beam.sdk.io.redis.RedisIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/redis/RedisIO.html + - transform: DynamoDBIO + description: Read from and write to [Amazon DynamoDB](https://aws.amazon.com/dynamodb/). + implementations: + - language: java + name: org.apache.beam.sdk.io.aws.dynamodb.DynamoDBIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.html + - language: java + name: org.apache.beam.sdk.io.aws2.dynamodb.DynamoDBIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.html + - transform: ClickHouseIO + description: Transform for writing to [ClickHouse](https://clickhouse.tech/). + implementations: + - language: java + name: org.apache.beam.sdk.io.clickhouse.ClickHouseIO + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.html + - transform: DatabaseIO + description: Package databaseio provides transformations and utilities to interact with a generic database / SQL API. + implementations: + - language: go + name: github.com/apache/beam/sdks/go/pkg/beam/io/databaseio + url: https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/databaseio + - name: Miscellaneous + description: Miscellaneous I/O sources. + rows: + - transform: FlinkStreamingImpulseSource + description: A PTransform that provides an unbounded, streaming source of empty byte arrays. This can only be used with the Flink runner. + implementations: + - language: py + name: apache_beam.io.flink.flink_streaming_impulse_source + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.flink.flink_streaming_impulse_source.html + - transform: GenerateSequence + description: Generates a bounded or unbounded stream of integers. + implementations: + - language: java + name: org.apache.beam.sdk.io.GenerateSequence + url: https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/GenerateSequence.html + - language: py + name: apache_beam.io.external.generate_sequence.GenerateSequence + url: https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.generate_sequence.html diff --git a/website/www/site/layouts/partials/section-menu/en/community.html b/website/www/site/layouts/partials/section-menu/en/community.html index b2cafee008f7..d90582ffe861 100644 --- a/website/www/site/layouts/partials/section-menu/en/community.html +++ b/website/www/site/layouts/partials/section-menu/en/community.html @@ -14,6 +14,7 @@

  • Integrations
  • Contact Us
  • Policies
  • +
  • Powered by Apache Beam
  • YouTube channel
  • Twitter Handle
  • In Person
  • diff --git a/website/www/site/layouts/partials/section-menu/en/documentation.html b/website/www/site/layouts/partials/section-menu/en/documentation.html index a44e0455a7d1..b83774d34f06 100644 --- a/website/www/site/layouts/partials/section-menu/en/documentation.html +++ b/website/www/site/layouts/partials/section-menu/en/documentation.html @@ -66,6 +66,18 @@
  • Using I/O transforms
  • Built-in I/O connectors
  • +
  • + Built-in I/O connector guides + +
  • + +
  • Developing new I/O connectors @@ -162,17 +174,17 @@
  • @@ -200,7 +212,7 @@
  • - +
  • Java @@ -212,18 +224,18 @@
  • @@ -261,7 +273,7 @@
  • - + @@ -276,6 +288,8 @@
  • Pipeline options
  • Custom I/O
  • Custom windows
  • +
  • BigQueryIO
  • +
  • AI Platform
  • diff --git a/website/www/site/layouts/shortcodes/code_sample.html b/website/www/site/layouts/shortcodes/code_sample.html new file mode 100644 index 000000000000..4a94cc2e5346 --- /dev/null +++ b/website/www/site/layouts/shortcodes/code_sample.html @@ -0,0 +1,23 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} +{{/* + This shortcode is used to fetch a piece of code with tags from Beam code. + In setupDockerContainer stage of build process, we run build_code_samples.sh script, which + copies Beam project files from which we take code snippets to website/www/site/code_samples + directory. We do that because Hugo can't access files which are outside of its project tree + (Hugo's root is the directory that contains config.toml). When copying, we name this files + according to convention path_to_file_filename.extension in order to avoid name conflicts. + .Get 0 references an argument passed to code_sample shortcode which is a path to file with + code snippet. So if we pass path/to/file/filename.py to code_sample, $path variable will + have value code_samples/path_to_file_filename.py which Hugo can access. + There should be no breaklines here to make sure the string results do not get impacts of newlines. +*/}}{{ $tag := .Get 1 }}{{ $path := printf "code_samples/%s" (replaceRE "/" "_" (.Get 0)) }}{{ $data := readFile $path }}{{ $matchRegex := printf "%s%s%s%s%s" "\\[START " $tag "]\n[\\s\\S]*?\n.*\\[END " $tag "]" }}{{ $match := index (findRE $matchRegex $data) 0 }}{{ $lines := split $match "\n" }}{{ $lineCount := len $lines }}{{ $cleanedLines := $lines | first (sub $lineCount 1) | last (sub $lineCount 2) }}{{ $firstLine := index $cleanedLines 0 }}{{ $numberOfWhitespaces := index (findRE "^\\s*" $firstLine) 0 | len }}{{ $unindentRegex := printf "%s%d%s" "^\\s{" $numberOfWhitespaces "}" }}{{ $unindentedLines := apply $cleanedLines "replaceRE" $unindentRegex "" "." }}{{ $result := delimit $unindentedLines "\n" }}{{ print $result }} diff --git a/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html b/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html index 317b06141c37..956446920c71 100644 --- a/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html +++ b/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html @@ -132,6 +132,11 @@ The degree of parallelism to be used when distributing operations onto workers. If the parallelism is not set, the configured Flink default is used, or 1 if none can be found. Default: -1 + + reIterableGroupByKeyResult + Flag indicating whether result of GBK needs to be re-iterable. Re-iterable result implies that all values for a single key must fit in memory as we currently do not support spilling to disk. + Default: false + reportCheckpointDuration If not null, reports the checkpoint duration of each ParDo stage in the provided metric namespace. diff --git a/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html b/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html index 4170e25c543c..9abe5086df74 100644 --- a/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html +++ b/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html @@ -132,6 +132,11 @@ The degree of parallelism to be used when distributing operations onto workers. If the parallelism is not set, the configured Flink default is used, or 1 if none can be found. Default: -1 + + re_iterable_group_by_key_result + Flag indicating whether result of GBK needs to be re-iterable. Re-iterable result implies that all values for a single key must fit in memory as we currently do not support spilling to disk. + Default: false + report_checkpoint_duration If not null, reports the checkpoint duration of each ParDo stage in the provided metric namespace. diff --git a/website/www/site/layouts/shortcodes/github_sample.html b/website/www/site/layouts/shortcodes/github_sample.html deleted file mode 100644 index db9fe9319687..000000000000 --- a/website/www/site/layouts/shortcodes/github_sample.html +++ /dev/null @@ -1,15 +0,0 @@ -{{/* - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. -*/}} -{{/* - This shortcode is used to fetch a piece of code with tags from GitHub. - There should be no breaklines here to make sure the string results do not get impacts of newlines. -*/}}{{ $tag := .Get 1 }}{{ $path := replaceRE "/blob/" "/" (.Get 0) }}{{ $path := printf "github_samples/%s" (replaceRE "/" "_" $path) }}{{ $data := readFile $path }}{{ $matchRegex := printf "%s%s%s%s%s" "\\[START " $tag "]\n[\\s\\S]*?\n.*\\[END " $tag "]" }}{{ $match := index (findRE $matchRegex $data) 0 }}{{ $lines := split $match "\n" }}{{ $lineCount := len $lines }}{{ $cleanedLines := $lines | first (sub $lineCount 1) | last (sub $lineCount 2) }}{{ $firstLine := index $cleanedLines 0 }}{{ $numberOfWhitespaces := index (findRE "^\\s*" $firstLine) 0 | len }}{{ $unindentRegex := printf "%s%d%s" "^\\s{" $numberOfWhitespaces "}" }}{{ $unindentedLines := apply $cleanedLines "replaceRE" $unindentRegex "" "." }}{{ $result := delimit $unindentedLines "\n" }}{{ print $result }} diff --git a/website/www/site/layouts/shortcodes/io-matrix.html b/website/www/site/layouts/shortcodes/io-matrix.html new file mode 100644 index 000000000000..3fd1eef025ac --- /dev/null +++ b/website/www/site/layouts/shortcodes/io-matrix.html @@ -0,0 +1,55 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. + */}} + +{{ $data := index $.Site.Data.io_matrix }} + +{{ define "language-row" }} + {{ if len (where .row.implementations "language" .language) }} + + + {{ .row.transform | markdownify }} + {{ if .row.docs }} (guide){{ else }}{{ end }} + + {{ .row.description | markdownify }} + + {{ range where .row.implementations "language" .language }} + + {{ else }} + N/A + {{ end }} + + + {{ else }} + {{ end }} +{{ end }} + + +{{ range $data.categories }} +

    {{ .name }}

    +

    {{ .description }}

    + + + + + + + {{ range .rows }} + {{ block "language-row" (dict "row" . "language" "java") }}{{ end }} + {{ block "language-row" (dict "row" . "language" "py") }}{{ end }} + {{ block "language-row" (dict "row" . "language" "go") }}{{ end }} + {{ end }} +
    NameDescription + Javadoc + pydoc + Godoc +
    +{{ end }} \ No newline at end of file diff --git a/website/www/site/static/.htaccess b/website/www/site/static/.htaccess index 2ccc960afe4e..916c7558eb66 100644 --- a/website/www/site/static/.htaccess +++ b/website/www/site/static/.htaccess @@ -21,4 +21,4 @@ RewriteRule ^(.*)$ https://beam.apache.org/$1 [L,R=301] # The following redirect maintains the previously supported URLs. RedirectMatch permanent "/documentation/sdks/(javadoc|pydoc)(.*)" "https://beam.apache.org/releases/$1$2" # Keep this updated to point to the current release. -RedirectMatch "/releases/([^/]+)/current(.*)" "https://beam.apache.org/releases/$1/2.20.0$2" +RedirectMatch "/releases/([^/]+)/current(.*)" "https://beam.apache.org/releases/$1/2.21.0$2" diff --git a/website/www/site/static/images/blog/beamsummit/beamsummit-digital-2020.png b/website/www/site/static/images/blog/beamsummit/beamsummit-digital-2020.png new file mode 100644 index 000000000000..36025d3e44f2 Binary files /dev/null and b/website/www/site/static/images/blog/beamsummit/beamsummit-digital-2020.png differ diff --git a/website/www/site/static/images/blog/beamsummit/beamsummit-digital.png b/website/www/site/static/images/blog/beamsummit/beamsummit-digital.png deleted file mode 100644 index a9d206a3f873..000000000000 Binary files a/website/www/site/static/images/blog/beamsummit/beamsummit-digital.png and /dev/null differ diff --git a/website/www/site/static/js/language-switch.js b/website/www/site/static/js/language-switch.js index 57d8f20341c0..7b6cb5942117 100644 --- a/website/www/site/static/js/language-switch.js +++ b/website/www/site/static/js/language-switch.js @@ -68,7 +68,7 @@ $(document).ready(function() { }, /** * @desc Search next sibling and if it's also a code block, then store - it's type and move onto the next element. It will keep + its type and move on to the next element. It will keep looking until there is no direct code block descendant left. * @param object $el - jQuery object, from where to start searching. * @param array $lang - list to hold types, found while searching.