Support Create and Update in FhirIO

.test-infra/jenkins/README.md

@@ -94,13 +94,16 @@ Beam Jenkins overview page: [link](https://builds.apache.org/view/A-D/view/Beam/
 |------|------|-------------------|-------------|
 | beam_PerformanceTests_Analysis | [cron](https://builds.apache.org/job/beam_PerformanceTests_Analysis/) | `Run Performance Tests Analysis` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Analysis/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Analysis) |
 | beam_PerformanceTests_AvroIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS/) | `Run Java AvroIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_AvroIOIT_HDFS) |
+| beam_PerformanceTests_BiqQueryIO_Read_Python | [cron](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python/) | `Run BigQueryIO Read Performance Test Python` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Read_Python) |
+| beam_PerformanceTests_BiqQueryIO_Write_Python_Batch | [cron](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch/) | `Run BigQueryIO Write Performance Test Python Batch` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch) |
 | beam_PerformanceTests_Compressed_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS/) | `Run Java CompressedTextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS) |
 | beam_PerformanceTests_HadoopFormat | [cron](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat/) | `Run Java HadoopFormatIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_HadoopFormat) |
 | beam_PerformanceTests_JDBC | [cron](https://builds.apache.org/job/beam_PerformanceTests_JDBC/) | `Run Java JdbcIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_JDBC/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_JDBC) |
 | beam_PerformanceTests_KafkaIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_Kafka_IO/) | `Run Java KafkaIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Kafka_IO/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Kafka_IO) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Kafka_IO/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Kafka_IO) |
 | beam_PerformanceTests_ManyFiles_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS/) | `Run Java ManyFilesTextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS) |
 | beam_PerformanceTests_MongoDBIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT/) | `Run Java MongoDBIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_MongoDBIO_IT) |
 | beam_PerformanceTests_ParquetIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS/) | `Run Java ParquetIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS) |
+| beam_PerformanceTests_PubsubIOIT_Python_Streaming | [cron](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming/) | `Run PubsubIO Performance Test Python` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming) |
 | beam_PerformanceTests_Spark | [cron](https://builds.apache.org/job/beam_PerformanceTests_Spark/) | `Run Spark Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_Spark/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_Spark) |
 | beam_PerformanceTests_TFRecordIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT/) | `Run Java TFRecordIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT) |
 | beam_PerformanceTests_TextIOIT | [cron](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT/), [hdfs_cron](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/) | `Run Java TextIO Performance Test` | [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT) [![Build Status](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/badge/icon)](https://builds.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS) |

.test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy

@@ -82,27 +82,27 @@ def executeJob = { scope, testConfig ->
 }
 
 PhraseTriggeringPostCommitBuilder.postCommitJob(
-        'beam_BiqQueryIO_Read_Performance_Test_Python',
+        'beam_PerformanceTests_BiqQueryIO_Read_Python',
         'Run BigQueryIO Read Performance Test Python',
         'BigQueryIO Read Performance Test Python',
         this
 ) {
     executeJob(delegate, bqio_read_test)
 }
 
-CronJobBuilder.cronJob('beam_BiqQueryIO_Read_Performance_Test_Python', 'H 15 * * *', this) {
+CronJobBuilder.cronJob('beam_PerformanceTests_BiqQueryIO_Read_Python', 'H 15 * * *', this) {
     executeJob(delegate, bqio_read_test)
 }
 
 PhraseTriggeringPostCommitBuilder.postCommitJob(
-        'beam_BiqQueryIO_Write_Performance_Test_Python_Batch',
+        'beam_PerformanceTests_BiqQueryIO_Write_Python_Batch',
         'Run BigQueryIO Write Performance Test Python Batch',
         'BigQueryIO Write Performance Test Python Batch',
         this
 ) {
     executeJob(delegate, bqio_write_test)
 }
 
-CronJobBuilder.cronJob('beam_BiqQueryIO_Write_Performance_Test_Python_Batch', 'H 15 * * *', this) {
+CronJobBuilder.cronJob('beam_PerformanceTests_BiqQueryIO_Write_Python_Batch', 'H 15 * * *', this) {
     executeJob(delegate, bqio_write_test)
 }

.test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy

@@ -39,7 +39,10 @@ def psio_test = [
                 temp_location             : 'gs://temp-storage-for-perf-tests/loadtests',
                 publish_to_big_query      : true,
                 metrics_dataset           : 'beam_performance',
-                metrics_table             : 'psio_io_2GB_msg_results',
+                metrics_table             : 'psio_io_2GB_results',
+                influx_measurement        : 'python_psio_2GB_results',
+                influx_db_name            : InfluxDBCredentialsHelper.InfluxDBDatabaseName,
+                influx_hostname           : InfluxDBCredentialsHelper.InfluxDBHostname,
                 input_options             : '\'{' +
                         '"num_records": 2097152,' +
                         '"key_size": 1,' +
@@ -59,14 +62,14 @@ def executeJob = { scope, testConfig ->
 }
 
 PhraseTriggeringPostCommitBuilder.postCommitJob(
-        'beam_PubsubIO_Performance_Test_Python',
+        'beam_PerformanceTests_PubsubIOIT_Python_Streaming',
         'Run PubsubIO Performance Test Python',
         'PubsubIO Performance Test Python',
         this
 ) {
     executeJob(delegate, psio_test)
 }
 
-CronJobBuilder.cronJob('beam_PubsubIO_Performance_Test_Python', 'H 15 * * *', this) {
+CronJobBuilder.cronJob('beam_PerformanceTests_PubsubIOIT_Python_Streaming', 'H 15 * * *', this) {
     executeJob(delegate, psio_test)
 }

.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy

@@ -39,5 +39,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
     shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python35')
     shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python36')
     shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python37')
+    // TODO(BEAM-9754): Turn on ValidatesContainer tests on Python 3.8 once BEAM-9754 is resolved.
+    // shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python38')
   }
 }

.test-infra/metrics/beamgrafana-deploy.yaml

@@ -37,7 +37,7 @@ spec:
         fsGroup: 1000
       containers:
       - name: beamgrafana
-        image: gcr.io/apache-beam-testing/beamgrafana:beammetrics20200515
+        image: gcr.io/apache-beam-testing/beamgrafana:beammetrics20200526
         securityContext:
           runAsUser: 0
         env:

.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_IO_IT_Tests_Dataflow.json

@@ -35,7 +35,7 @@
       },
       "hiddenSeries": false,
       "id": 2,
-      "interval": "",
+      "interval": "24h",
       "legend": {
         "avg": false,
         "current": false,
@@ -75,7 +75,7 @@
           "measurement": "python_bqio_read",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT value FROM \"python_bqio_read_10GB_results\" WHERE \"metric\" =~ /runtime/",
+          "query": "SELECT mean(\"value\") FROM \"python_bqio_read_10GB_results\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
           "rawQuery": true,
           "refId": "A",
           "resultFormat": "time_series",
@@ -157,7 +157,7 @@
       },
       "hiddenSeries": false,
       "id": 3,
-      "interval": "",
+      "interval": "24h",
       "legend": {
         "avg": false,
         "current": false,
@@ -197,7 +197,7 @@
           "measurement": "python_bqio_read",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT value FROM \"python_bqio_write_10GB_results\" WHERE \"metric\" =~ /runtime/",
+          "query": "SELECT mean(\"value\") FROM \"python_bqio_write_10GB_results\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
           "rawQuery": true,
           "refId": "A",
           "resultFormat": "time_series",
@@ -261,6 +261,250 @@
         "align": false,
         "alignLevel": null
       }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "BeamInfluxDB",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "interval": "24h",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pluginVersion": "6.7.2",
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "alias": "read_time",
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            }
+          ],
+          "measurement": "python_bqio_read",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT mean(\"value\") FROM \"python_psio_2GB_results\" WHERE \"metric\" = 'pubsub_io_perf_read_runtime' AND $timeFilter GROUP BY time($__interval), \"metric\"",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Reading 2GB of data | Pubsub native Dataflow IO | streaming",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "transparent": true,
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:403",
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:404",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "BeamInfluxDB",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "interval": "24h",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pluginVersion": "6.7.2",
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "alias": "write_time",
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            }
+          ],
+          "measurement": "python_bqio_read",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT mean(\"value\") FROM \"python_psio_2GB_results\" WHERE \"metric\" = 'pubsub_io_perf_write_runtime' AND $timeFilter GROUP BY time($__interval), \"metric\"",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Writing 2GB of data | Pubsub native Dataflow IO | streaming",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "transparent": true,
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:403",
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:404",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
     }
   ],
   "schemaVersion": 22,
@@ -293,5 +537,5 @@
   "variables": {
     "list": []
   },
-  "version": 1
-}
+  "version": 2
+}