TDL-26703: Refactor entire tap

.circleci/config.yml

@@ -0,0 +1,61 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester-uv
+    steps:
+      - checkout
+      - run:
+          name: 'Setup virtual env'
+          command: |
+            uv venv --python 3.9 /usr/local/share/virtualenvs/tap-chargebee
+            source /usr/local/share/virtualenvs/tap-chargebee/bin/activate
+            uv pip install -U 'pip<19.2' 'setuptools<51.0.0'
+            uv pip install .[dev]
+      - run:
+          name: 'JSON Validator'
+          command: |
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            stitch-validate-json tap_chargebee/schemas/*/*.json
+      - run:
+          name: 'Pylint'
+          command: |
+            source /usr/local/share/virtualenvs/tap-chargebee/bin/activate
+            uv pip install pylint==2.14.1
+            pylint tap_chargebee --disable C,W,R,no-member
+      - run:
+          name: 'Unit Tests'
+          command: |
+            source /usr/local/share/virtualenvs/tap-chargebee/bin/activate
+            uv pip install pytest coverage
+            coverage run -m pytest tests/unittests
+            coverage html
+      - store_test_results:
+          path: test_output/report.xml
+      - store_artifacts:
+          path: htmlcov
+      - run:
+          name: 'Integration Tests'
+          command: |
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            uv pip install --upgrade awscli
+            aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
+            source dev_env.sh
+            run-test --tap=tap-chargebee tests
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build:
+          context: circleci-user
+  build_daily:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - build:
+          context: circleci-user

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+# Description of change
+(write a short description or paste a link to JIRA)
+
+# Manual QA steps
+ - 
+
+# Risks
+ - 
+
+# Rollback steps
+ - revert this branch
+
+#### AI generated code
+https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code
+- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool

CHANGELOG.md

@@ -1,4 +1,101 @@
 # Changelog
 
+## 1.4.0
+
+- Added discounts field in subscriptions schema #[107](https://github.com/singer-io/tap-chargebee/pull/107)
+- Fixed All fields & Pagination integration tests #[109](https://github.com/singer-io/tap-chargebee/pull/109)
+
+## 1.3.5
+
+- Add missing Business Entity Details from Chargebee customers details #[100](https://github.com/singer-io/tap-chargebee/pull/100)
+- Prevent truncating invoice lineItem taxRates #[102](https://github.com/singer-io/tap-chargebee/pull/102)
+
+## 1.3.4
+
+- Custom field support for Item Model entities #[92](https://github.com/singer-io/tap-chargebee/pull/92)
+- Updated integration test #[93](https://github.com/singer-io/tap-chargebee/pull/93)
+
+## 1.3.3
+
+- Implement request timeout #[78](https://github.com/singer-io/tap-chargebee/pull/78)
+- Add missing tap-tester tests #[83](https://github.com/singer-io/tap-chargebee/pull/83)
+- Add custom exception handling #[85](https://github.com/singer-io/tap-chargebee/pull/85)
+- Add missing fields to schema #[87](https://github.com/singer-io/tap-chargebee/pull/87)
+- Revert back bookmark logic #[88](https://github.com/singer-io/tap-chargebee/pull/88)
+
+##1.3.2
+
+- Revert back bookmarking logic [#86](https://github.com/singer-io/tap-chargebee/pull/86)
+
+## 1.3.1
+
+- Added support for Chargebee Quotes [#75](https://github.com/singer-io/tap-chargebee/pull/75)
+
+## 1.3.0
+
+- Added comments stream [#52](https://github.com/singer-io/tap-chargebee/pull/52)
+- Added include_deleted configuration [#58](https://github.com/singer-io/tap-chargebee/pull/58)
+- Added undocumented fields [#69](https://github.com/singer-io/tap-chargebee/pull/69)
+
+## 1.2.2
+
+- Update the schema glob so that we include all schemas in the package distribution [#73](https://github.com/singer-io/tap-chargebee/pull/73)
+
+## 1.2.1
+
+- Add a `MANIFEST.in` file to include schema files in the `tap-chargebee` package [#72](https://github.com/singer-io/tap-chargebee/pull/72)
+
+## 1.2.0
+
+- Remove all minimum/maximum and minLength/maxLength [#45][#45]
+- Fix JSONDecodeError in Invoices and Transactions streams [#51][#51]
+- Add Tiersprice attribute [#53][#53]
+- Updated integration test to cover product catalog v1 and v2 [#63][#63]
+- Add additional fields from API [#64][#64]
+- Upgraded event stream schema [#57][#57]
+- Updated Bookmark handling, date without tz will updated in UTC tz format [#54][#54]
+
+[#45]: https://github.com/singer-io/tap-chargebee/pull/45
+[#51]: https://github.com/singer-io/tap-chargebee/pull/51
+[#53]: https://github.com/singer-io/tap-chargebee/pull/53
+[#63]: https://github.com/singer-io/tap-chargebee/pull/63
+[#64]: https://github.com/singer-io/tap-chargebee/pull/64
+[#57]: https://github.com/singer-io/tap-chargebee/pull/57
+[#54]: https://github.com/singer-io/tap-chargebee/pull/54
+
+## 1.1.2
+
+- Fix domain name comparison bug [#67](https://github.com/singer-io/tap-chargebee/pull/67)
+
+## 1.1.1
+
+- Add an error message when we get an unexpected response from the Configurations API [#62](https://github.com/singer-io/tap-chargebee/pull/62)
+
+## 1.1.0
+
+- Adds support for Item Model, Multi-decimal (for Plan Model), and Account hierarchy (for Plan Model) [#56](https://github.com/singer-io/tap-chargebee/pull/56)
+- Organized the folder structure:
+  a. common(common schemas to both plan model and item model)
+  b. item_model
+  c. plan_model
+- Introduces two new streams: ITEM_MODEL_AVAILABLE_STREAMS, PLAN_MODEL_AVAILABLE_STREAMS
+
+## 1.0.3
+
+- Fix invalid JSON from #44
+
+## 1.0.2
+
+- Remove `maxLength` from `payment_sources` schema to address certain integrations having IDs of greater length than specified, and make the schema more flexible as the API evolves [#44](https://github.com/singer-io/tap-chargebee/pull/44)
+
+## 1.0.0
+
+- No change from 0.0.12
+
+## 0.0.12
+
+- Add `custom_fields` to plans, addons, customers, and subscriptions [#9](https://github.com/singer-io/tap-chargebee/pull/9)
+
 ## 0.0.3
-  * Add `credit_notes` stream [#2](https://github.com/singer-io/tap-chargebee/pull/2)
+
+- Add `credit_notes` stream [#2](https://github.com/singer-io/tap-chargebee/pull/2)

MANIFEST.in

@@ -0,0 +1 @@
+include tap_chargebee/schemas/*/*.json

README.md

@@ -11,9 +11,12 @@ This tap:
   - [Addons](https://apidocs.chargebee.com/docs/api/addons)
   - [Coupons](https://apidocs.chargebee.com/docs/api/coupons)
   - [Credit Notes](https://apidocs.chargebee.com/docs/api/credit_notes)
+  - [Comments](https://apidocs.chargebee.com/docs/api/comments)  
   - [Customers](https://apidocs.chargebee.com/docs/api/customers)
   - [Events](https://apidocs.chargebee.com/docs/api/events)
+  - [Gifts](https://apidocs.chargebee.com/docs/api/gifts)
   - [Invoices](https://apidocs.chargebee.com/docs/api/invoices)
+  - [Orders](https://apidocs.chargebee.com/docs/api/orders)
   - [Payment Sources](https://apidocs.chargebee.com/docs/api/payment_sources)
   - [Plans](https://apidocs.chargebee.com/docs/api/plans)
   - [Subscriptions](https://apidocs.chargebee.com/docs/api/subscriptions)
@@ -36,19 +39,25 @@ This tap:
 
    ```json
     {
-        "start_date": "2010-01-01",
+        "start_date": "2010-01-01T00:00:00Z",
         "api_key": "<Chargebee API Key>",
-        "site": "<Chargebee Site>"
+        "site": "<Chargebee Site>",
+        "include_deleted": "True|False",
+        "request_timeout": 300
     }
     ```
 
-   The `start_date` specifies the date at which the tap will begin pulling data
+   The `start_date` specifies the date in ISO(YYYY-mm-ddTHH:MM:SSZ) format at which the tap will begin pulling data
    (for those resources that support this).
 
    The `api_key` is the API key for your Chargebee site.
 
    The `site` parameter represents the name of your specific Chargebee site (e.g. `https://{site}.chargebee.com/api/v2/subscriptions`)
 
+   The `include_deleted` is an optional flag to ask if you want deleted records of all streams or not. Default: true 
+
+   The `request_timeout` is an optional paramater to set timeout for requests. Default: 300 seconds
+
 4. Run the Tap in Discovery Mode
 
     ```bash

setup.py

@@ -3,13 +3,15 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-chargebee',
-      version='0.0.5',
+      version='1.4.0',
       description='Singer.io tap for extracting data from the Chargebee API',
       author='dwallace@envoy.com',
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_chargebee'],
       install_requires=[
-          'tap-framework==0.1.0'
+          'singer-python==6.0.0',
+          'backoff==2.2.1',
+          'requests==2.31.0'
       ],
       entry_points='''
           [console_scripts]
@@ -18,6 +20,10 @@
       packages=find_packages(),
       package_data={
           'tap_chargebee': [
-              'schemas/*.json'
+              'schemas/common/*.json',
+              'schemas/item_model/*.json',
+              'schemas/plan_model/*.json'
           ]
-      })
+      },
+      include_package_data=True)
+

tap_chargebee/__init__.py

@@ -1,32 +1,122 @@
 import singer
-import tap_framework
-import tap_chargebee.client
-import tap_chargebee.streams
+import sys
+import json
+from singer import metadata, Catalog
+from tap_chargebee.client import ChargebeeClient
+import tap_chargebee.streams as streams
 
 LOGGER = singer.get_logger()
 
 
-class ChargebeeRunner(tap_framework.Runner):
-    pass
+def stream_is_selected(mdata):
+    """
+    Check if the stream is selected
+    """
+    return mdata.get((), {}).get("selected", False)
+
+
+def get_available_streams(config: dict, cb_client: ChargebeeClient):
+    """
+    Prepare the available streams based on the product catalog version
+    """
+    site_name = config.get("site")
+    LOGGER.info("Site Name %s", site_name)
+    configuration_url = f"https://{site_name}.chargebee.com/api/v2/configurations"
+
+    try:
+        # Make a request to the configurations API
+        response = cb_client.make_request(url=configuration_url, method="GET")
+        site_configurations = response["configurations"]
+    except Exception as e:
+        LOGGER.error("Failed to fetch configurations: %s", e)
+        raise e
+
+    # Fetch the product catalog version
+    product_catalog_version = next(
+        iter(
+            config["product_catalog_version"]
+            for config in site_configurations
+            if config["domain"].lower() == site_name.lower()
+        ),
+        None,
+    )
+
+    if product_catalog_version == "v2":
+        available_streams = streams.ITEM_MODEL_AVAILABLE_STREAMS
+        config["item_model"] = True
+        LOGGER.info("Found product catalog version v2")
+    elif product_catalog_version == "v1":
+        available_streams = streams.PLAN_MODEL_AVAILABLE_STREAMS
+        config["item_model"] = False
+        LOGGER.info("Found product catalog version v1")
+    else:
+        LOGGER.error(
+            "Incorrect Product Catalog version {}".format(product_catalog_version)
+        )
+        raise RuntimeError("Incorrect Product Catalog version")
+
+    return available_streams
+
+
+def do_discover(config: dict, state: dict, available_streams: list):
+    """
+    Generate the catalog
+    """
+    LOGGER.info("Starting discovery.")
+    catalog = []
+
+    # Generate catalog for each stream based on the product catalog version
+    for available_stream in available_streams:
+        stream = available_stream(config, state, None, None)
+        catalog += stream.generate_catalog()
+
+    json.dump({"streams": catalog}, sys.stdout, indent=4)
+    LOGGER.info("Finished discover mode")
+
+
+def do_sync(config: dict, catalog: Catalog, state: dict, client: ChargebeeClient):
+    """
+    Sync data from Chargebee.
+    """
+
+    last_stream = singer.get_currently_syncing(state)
+    LOGGER.info("last/currently syncing stream: %s", last_stream)
+
+    # Resume sync from the last stream
+    for catalog_entry in catalog.get_selected_streams(state):
+        mdata = metadata.to_map(catalog_entry.metadata)
+        replication_key = metadata.get(mdata, (), "replication-key")
+        key_properties = metadata.get(mdata, (), "table-key-properties")
+        stream_name = catalog_entry.tap_stream_id
+
+        singer.set_currently_syncing(state, stream_name)
+        singer.write_state(state)
+        singer.write_schema(
+            stream_name, catalog_entry.schema.to_dict(), key_properties, replication_key
+        )
+
+        LOGGER.info("%s: Starting sync", stream_name)
+        instance = streams.STREAMS[stream_name](config, state, catalog_entry, client)
+        counter_value = instance.sync()
+        singer.set_currently_syncing(state, None)
+        singer.write_state(state)
+        LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter_value)
 
 
 @singer.utils.handle_top_exception(LOGGER)
 def main():
     args = singer.utils.parse_args(
-        required_config_keys=['api_key', 'start_date', 'site'])
+        required_config_keys=["api_key", "start_date", "site"]
+    )
 
-    client = tap_chargebee.client.ChargebeeClient(args.config)
-
-    runner = ChargebeeRunner(
-        args, client, tap_chargebee.streams.AVAILABLE_STREAMS
-        )
+    client = ChargebeeClient(args.config)
+    available_streams = get_available_streams(args.config, client)
 
     if args.discover:
-        runner.do_discover()
-    else:
-        # import pdb; pdb.set_trace()
-        runner.do_sync()
+        do_discover(args.config, args.state, available_streams)
+    elif args.catalog:
+        do_sync(args.config, args.catalog, args.state, client)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()