From 470823d9dd6cf7728e04ec9c121fb375cdb9e369 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 12:45:50 +0530 Subject: [PATCH 001/184] Onboarded Asset V4 Apis --- ansible/roles/kong-api/defaults/main.yml | 108 +++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 0daa1bbbb..0bf47f3e9 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5257,3 +5257,111 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: createAsset + uris: "{{ asset_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/asset/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateAsset + uris: "{{ asset_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/asset/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readAsset + uris: "{{ asset_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/asset/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: uploadAsset + uris: "{{ asset_prefix }}/v1/upload" + upstream_url: "{{ content_service_url }}/asset/v4/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: uploadUrlAsset + uris: "{{ asset_prefix }}/v1/upload/url" + upstream_url: "{{ content_service_url }}/asset/v4/upload/url" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: retireAsset + uris: "{{ asset_prefix }}/v1/retire" + upstream_url: "{{ content_service_url }}/asset/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" From 10827f1b01735bd7493c0ea3a6ea2f30183a1176 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 13:17:50 +0530 Subject: [PATCH 002/184] Added changes for asset prefix --- ansible/roles/kong-api/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 0bf47f3e9..bdf576b2d 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -75,6 +75,8 @@ auth_service_prefix: /auth object_category_prefix: /object/category object_category_definition_prefix: /object/category/definition dataset_service_prefix: /dataset +asset_prefix: /asset + # Service URLs knowledge_mw_service_url: "http://knowledge-mw-service:5000" From 6768114fd04242a2933ec4f6a4516626f3d7344f Mon Sep 17 00:00:00 2001 From: NehaVerma Date: Tue, 13 Oct 2020 13:34:27 +0530 Subject: [PATCH 003/184] SB-21000 : Certificate Template mapping added --- .../stack-sunbird/templates/content-service_application.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf index 07dbcb6c0..a96cdc908 100644 --- a/ansible/roles/stack-sunbird/templates/content-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf @@ -534,6 +534,7 @@ contentTypeToPrimaryCategory { LessonPlanUnit: "Lesson Plan Unit" CourseUnit: "Course Unit" TextBookUnit: "Textbook Unit" + Asset: "Certificate Template" } resourceTypeToPrimaryCategory { From c7a3fcd7d65defafd4130e6ae4fb203ed9d5f6a8 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 14:10:22 +0530 Subject: [PATCH 004/184] Added fix for indentation issue --- ansible/roles/kong-api/defaults/main.yml | 192 +++++++++++------------ 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index bdf576b2d..7fccebf6f 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5261,109 +5261,109 @@ kong_apis: config.allowed_payload_size: "{{ small_request_size_limit }}" - name: createAsset - uris: "{{ asset_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/asset/v4/create" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/asset/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: updateAsset - uris: "{{ asset_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/asset/v4/update" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/asset/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: readAsset - uris: "{{ asset_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/asset/v4/read" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/asset/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: uploadAsset - uris: "{{ asset_prefix }}/v1/upload" - upstream_url: "{{ content_service_url }}/asset/v4/upload" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/upload" + upstream_url: "{{ content_service_url }}/asset/v4/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: uploadUrlAsset - uris: "{{ asset_prefix }}/v1/upload/url" - upstream_url: "{{ content_service_url }}/asset/v4/upload/url" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/upload/url" + upstream_url: "{{ content_service_url }}/asset/v4/upload/url" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: retireAsset - uris: "{{ asset_prefix }}/v1/retire" - upstream_url: "{{ content_service_url }}/asset/v4/retire" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + uris: "{{ asset_prefix }}/v1/retire" + upstream_url: "{{ content_service_url }}/asset/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" From 74c5b2c49013106026ed19f5b96e440751a0a2c6 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 16:59:14 +0530 Subject: [PATCH 005/184] Added collection v4 apis --- ansible/roles/kong-api/defaults/main.yml | 218 +++++++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 7fccebf6f..3e8c2d865 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -76,6 +76,7 @@ object_category_prefix: /object/category object_category_definition_prefix: /object/category/definition dataset_service_prefix: /dataset asset_prefix: /asset +collection_prefix: /collection # Service URLs @@ -5367,3 +5368,220 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionCreate + uris: "{{ collection_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/collection/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionRead + uris: "{{ collection_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/collection/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionUpdate + uris: "{{ collection_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/collection/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionFlag + uris: "{{ collection_prefix }}/v1/flag" + upstream_url: "{{ content_service_url }}/collection/v4/flag" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionDiscard + uris: "{{ collection_prefix }}/v1/discard" + upstream_url: "{{ content_service_url }}/collection/v4/discard" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionRetire + uris: "{{ collection_prefix }}/v1/retire" + upstream_url: "{{ content_service_url }}/collection/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyAdd + uris: "{{ collection_prefix }}/v1/hierarchy/add" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/add" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyRemove + uris: "{{ collection_prefix }}/v1/hierarchy/remove" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/remove" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyUpdate + uris: "{{ collection_prefix }}/v1/hierarchy/update" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyRead + uris: "{{ collection_prefix }}/v1/hierarchy" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + + - name: collectionDialcodeLink + uris: "{{ collection_prefix }}/v1/dialcode/link" + upstream_url: "{{ content_service_url }}/collection/v4/dialcode/link" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionCopy + uris: "{{ collection_prefix }}/v1/copy" + upstream_url: "{{ content_service_url }}/collection/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" From f065ce699b2c8fa003187da6f3a9be947b536ecc Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 17:29:48 +0530 Subject: [PATCH 006/184] Added content v4 apis --- ansible/roles/kong-api/defaults/main.yml | 216 +++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 3e8c2d865..87c65505b 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5585,3 +5585,219 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentCreate + uris: "{{ content_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/content/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentUpdate + uris: "{{ content_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/content/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentRead + uris: "{{ content_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/content/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentUpload + uris: "{{ content_prefix }}/v1/upload" + upstream_url: "{{ content_service_url }}/content/v4/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentUploadUrl + uris: "{{ content_prefix }}/v1/upload/url" + upstream_url: "{{ content_service_url }}/content/v4/upload/url" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentCopy + uris: "{{ content_prefix }}/v1/copy" + upstream_url: "{{ content_service_url }}/content/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentDialcodeLink + uris: "{{ content_prefix }}/v1/dialcode/link" + upstream_url: "{{ content_service_url }}/content/v4/dialcode/link" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentImport + uris: "{{ content_prefix }}/v1/import" + upstream_url: "{{ content_service_url }}/content/v4/import" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentFlag + uris: "{{ content_prefix }}/v1/flag" + upstream_url: "{{ content_service_url }}/content/v4/flag" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentFlagAccept + uris: "{{ content_prefix }}/v1/flag/accept" + upstream_url: "{{ content_service_url }}/content/v4/flag/accept" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentDiscard + uris: "{{ content_prefix }}/v1/flag/discard" + upstream_url: "{{ content_service_url }}/content/v4/flag/discard" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: contentRetire + uris: "{{ content_prefix }}/v1/flag/retire" + upstream_url: "{{ content_service_url }}/content/v4/flag/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" \ No newline at end of file From f25f71ab506a0b74d905b8eaab70d5930a5c5544 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 17:33:04 +0530 Subject: [PATCH 007/184] Added content v4 apis --- ansible/roles/kong-api/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 87c65505b..5dc98bf3a 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5613,8 +5613,8 @@ kong_apis: - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: - - 'contentCreate' + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" From 22ef32dcaaeb48c870b0db2d19b65e61c0994280 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 17:44:38 +0530 Subject: [PATCH 008/184] Added changes to indentation --- ansible/roles/kong-api/defaults/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 5dc98bf3a..05700d2eb 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5739,8 +5739,8 @@ kong_apis: - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: - - 'contentCreate' + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5757,8 +5757,8 @@ kong_apis: - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: - - 'contentCreate' + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" From e236b481ea2e9ae2a6974c6a9d4c186848ca3be1 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 17:54:34 +0530 Subject: [PATCH 009/184] Added changes to indentation --- ansible/roles/kong-api/defaults/main.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 05700d2eb..e452265cf 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5717,18 +5717,18 @@ kong_apis: upstream_url: "{{ content_service_url }}/content/v4/import" strip_uri: true plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" - name: contentFlag uris: "{{ content_prefix }}/v1/flag" From 87b9e0838015caae80e5d280bf3c78a7a3749c34 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Tue, 13 Oct 2020 18:02:01 +0530 Subject: [PATCH 010/184] Removed content onboarded v4 api's --- ansible/roles/kong-api/defaults/main.yml | 216 ----------------------- 1 file changed, 216 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index e452265cf..3e8c2d865 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5585,219 +5585,3 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentCreate - uris: "{{ content_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/content/v4/create" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentUpdate - uris: "{{ content_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/content/v4/update" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentRead - uris: "{{ content_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/content/v4/read" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentUpload - uris: "{{ content_prefix }}/v1/upload" - upstream_url: "{{ content_service_url }}/content/v4/upload" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentUploadUrl - uris: "{{ content_prefix }}/v1/upload/url" - upstream_url: "{{ content_service_url }}/content/v4/upload/url" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentCopy - uris: "{{ content_prefix }}/v1/copy" - upstream_url: "{{ content_service_url }}/content/v4/copy" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentDialcodeLink - uris: "{{ content_prefix }}/v1/dialcode/link" - upstream_url: "{{ content_service_url }}/content/v4/dialcode/link" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentImport - uris: "{{ content_prefix }}/v1/import" - upstream_url: "{{ content_service_url }}/content/v4/import" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentFlag - uris: "{{ content_prefix }}/v1/flag" - upstream_url: "{{ content_service_url }}/content/v4/flag" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentFlagAccept - uris: "{{ content_prefix }}/v1/flag/accept" - upstream_url: "{{ content_service_url }}/content/v4/flag/accept" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentDiscard - uris: "{{ content_prefix }}/v1/flag/discard" - upstream_url: "{{ content_service_url }}/content/v4/flag/discard" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - - name: contentRetire - uris: "{{ content_prefix }}/v1/flag/retire" - upstream_url: "{{ content_service_url }}/content/v4/flag/retire" - strip_uri: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: - - 'contentCreate' - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" \ No newline at end of file From d15877765722d3dee275c2d9b950c32b1cfd99a9 Mon Sep 17 00:00:00 2001 From: satish Nirmalkar Date: Tue, 13 Oct 2020 20:36:48 +0530 Subject: [PATCH 011/184] SB-20892 Onboarding dlete api --- ansible/roles/kong-api/defaults/main.yml | 18 ++++++++++++++++++ ansible/roles/kong-consumer/defaults/main.yml | 2 ++ 2 files changed, 20 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 0daa1bbbb..84979c0d7 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -4754,6 +4754,24 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: deleteGroup + uris: "{{ group_service_prefix }}/v1/delete" + upstream_url: "{{ group_service_url }}/v1/group/delete" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'groupAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: groupActivityAgg uris: "{{ data_service_prefix }}/v1/group/activity/agg" upstream_url: "{{ lms_service_url }}/v1/group/activity/agg" diff --git a/ansible/roles/kong-consumer/defaults/main.yml b/ansible/roles/kong-consumer/defaults/main.yml index 77caf0ff4..2ab952531 100644 --- a/ansible/roles/kong-consumer/defaults/main.yml +++ b/ansible/roles/kong-consumer/defaults/main.yml @@ -25,6 +25,7 @@ mobile_device_groups: - groupCreate - groupUpdate - groupAccess + - groupAdmin - refreshTokenCreate kong_all_consumer_groups: @@ -79,6 +80,7 @@ kong_all_consumer_groups: - groupCreate - groupUpdate - groupAccess + - groupAdmin - dataAccess - druidReportCreate - druidReportAdmin From 8b156a22f991a99585ad9462966f75076ec6298f Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Wed, 14 Oct 2020 18:27:35 +0530 Subject: [PATCH 012/184] Added changes to the acl as per requested --- ansible/roles/kong-api/defaults/main.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 3e8c2d865..8a52df87f 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5289,7 +5289,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5307,7 +5307,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5361,7 +5361,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5397,7 +5397,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5415,7 +5415,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5433,7 +5433,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5451,7 +5451,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5469,7 +5469,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5487,7 +5487,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5505,7 +5505,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5523,7 +5523,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5541,7 +5541,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" From b5753b6273aec3bf56b5c0a8ab515abbe20f51e8 Mon Sep 17 00:00:00 2001 From: rheafernandes Date: Wed, 14 Oct 2020 18:51:45 +0530 Subject: [PATCH 013/184] Added changes to the acl as per requested --- ansible/roles/kong-api/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 8a52df87f..5cb15cf39 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -5397,7 +5397,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentUpdate' + - 'contentAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -5560,7 +5560,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'contentCreate' + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" From fdecf11afab7ec3b867494eccfbe916b4580899a Mon Sep 17 00:00:00 2001 From: SMY ALTAMASH <30286162+SMYALTAMASH@users.noreply.github.com> Date: Thu, 15 Oct 2020 12:02:30 +0530 Subject: [PATCH 014/184] Degrading the permission from admin to search to be used wity mobile devices. (#1989) --- ansible/roles/kong-api/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 25ad09a48..4fda41404 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -4587,7 +4587,7 @@ kong_apis: - "{{ statsd_pulgin }}" - name: acl config.whitelist: - - 'certificateAdmin' + - 'certificateAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" From 6b384c1af02af97fa2034d489f3dec773a66a0a3 Mon Sep 17 00:00:00 2001 From: kaliraja <34502260+Kaali09@users.noreply.github.com> Date: Tue, 20 Oct 2020 15:48:16 +0530 Subject: [PATCH 015/184] added the druid alert rules (#1996) * Issue #TG607: added the druid alert rules * Issue #TG607: added the druid alert rules * Issue #TG607: updated the druid alert rules * Issue #0000: added the datasource name in message field. * Issue #0000: removed the rollup datasource --- .../templates/alertrules.yaml | 16 +++ .../alertrules/templates/promrulesDruid.yml | 103 ++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 kubernetes/helm_charts/monitoring/alertrules/templates/promrulesDruid.yml diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml index a646b18f9..5a6466670 100644 --- a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml @@ -43,3 +43,19 @@ kafka_telemetry_ingestion_backup_threshold_critical: "{{ kafka_telemetry_ingesti kafka_telemetry_unique_backup_threshold_critical: "{{ kafka_telemetry_unique_backup_threshold_critical }}" summary_backup_threshold_critical: "{{ summary_backup_threshold_critical }}" summary_channel_backup_threshold_critical: "{{ summary_channel_backup_threshold_critical }}" + + +raw_druid_datasource: + - content-model-snapshot + - offline-desktop-events + - summary-events + - telemetry-events-syncts + - telemetry-feedback-events + - audit-rollup-syncts + - sl_observation_status + - sl_observations_production + - telemetry-feedback-events + +druid_raw_ingestion_threshold: 200000 +druid_rollup_ingestion_threshold: 200000 + diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesDruid.yml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesDruid.yml new file mode 100644 index 000000000..d6b702b21 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesDruid.yml @@ -0,0 +1,103 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + role: alert-rules + app: {{ .Values.prometheus_rule_selector_app }} + release: {{ .Values.prometheus_rule_selector_release }} + name: {{ .Values.fullnameOverride }}-druid-rules + namespace: {{ .Values.namespace }} +spec: + groups: + - name: alertrules.process + rules: + {{- if .Values.raw_druid_datasource }} + ## druid segment unavailable count alert + {{ range $key := .Values.raw_druid_datasource }} + - alert: druid_segment_unavailable_count + expr: druid_segment_unavailable_count{exported_job={{ . | quote }}} > 0 + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid segment unavailable count for the datasource {{ . }} is greater than 0. + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidSegmentUnavailable + {{- end }} + ### kafka druid ingest lag + {{ range $key := .Values.raw_druid_datasource }} + - alert: druid_ingest_kafka_lag + expr: druid_ingest_kafka_lag{exported_job={{ . | quote }}} > {{ $.Values.druid_raw_ingestion_threshold }} + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid ingestion lag for datasource {{ . }} is greater than threshold {{ $.Values.druid_raw_ingestion_threshold | int }} + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidKafkaIngestionLag + {{- end }} + ### druid ingest handoff failed alert rules + {{ range $key := .Values.raw_druid_datasource }} + - alert: druid_ingest_handoff_failed + expr: druid_ingest_handoff_failed{exported_job={{ . | quote }}} > 0 + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid ingest handoff failed count for datasource {{ . }} is greater than 0. + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidIngestHandoffFailed + {{- end }} + {{- end }} + + {{- if .Values.rollup_druid_cluster }} + ## druid segment unavailable count alert + {{ range $key := .Values.rollup_druid_datasource }} + - alert: druid_segment_unavailable_count + expr: druid_segment_unavailable_count{exported_job={{ . | quote }}} > 0 + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid segment unavailable count for the datasource {{ . }} is greater than 0. + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidSegmentUnavailable + {{- end }} + ### kafka druid ingest lag + {{ range $key := .Values.rollup_druid_datasource }} + - alert: druid_ingest_kafka_lag + expr: druid_ingest_kafka_lag{exported_job={{ . | quote }}} > {{ $.Values.druid_rollup_ingestion_threshold }} + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid ingestion lag for datasource {{ . }} is greater than {{ $.Values.druid_rollup_ingestion_threshold | int }} + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidKafkaIngestionLag + {{- end }} + ### druid ingest handoff failed alert rules + {{ range $key := .Values.rollup_druid_datasource }} + - alert: druid_ingest_handoff_failed + expr: druid_ingest_handoff_failed{exported_job={{ . | quote }}} > 0 + for: 5m + labels: + severity: critical + module: dp_lag + annotations: + message: The druid ingest handoff failed count for datasource {{ . }} is greater than 0. + lag: {{`({{ humanize $value }})`}} + job_id: Druid + alertname: DruidIngestHandoffFailed + {{- end }} + {{- end }} From 715a87c6e9b562e8fee65df78a8a6d97d9e12c33 Mon Sep 17 00:00:00 2001 From: Hari-stackroute <40484996+Hari-stackroute@users.noreply.github.com> Date: Thu, 22 Oct 2020 13:27:57 +0530 Subject: [PATCH 016/184] SB-20938 added create and update feed api details (#1998) --- ansible/roles/kong-api/defaults/main.yml | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 4dee3c646..b615a9eac 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -4343,6 +4343,42 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: userFeedCreate + uris: "{{ user_service_prefix }}/feed/v1/create" + upstream_url: "{{ learning_service_url }}/v1/user/feed/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'userAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: userFeedUpdate + uris: "{{ user_service_prefix }}/feed/v1/update" + upstream_url: "{{ learning_service_url }}/v1/user/feed/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'userAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: userMigrate uris: "{{ user_service_prefix }}/v1/migrate" upstream_url: "{{ learning_service_url }}/v1/user/migrate" From f95f3782129299bba4bb78662e1611d1ed3b209e Mon Sep 17 00:00:00 2001 From: sknirmalkar89 <59080585+sknirmalkar89@users.noreply.github.com> Date: Thu, 22 Oct 2020 15:00:50 +0530 Subject: [PATCH 017/184] SC-2059 Adding new membership update api (#1999) --- ansible/roles/kong-api/defaults/main.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index b615a9eac..4d533ea44 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -4829,6 +4829,24 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: updateGroupMembership + uris: "{{ group_service_prefix }}/membership/v1/update" + upstream_url: "{{ group_service_url }}/v1/group/membership/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'groupUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: groupActivityAgg uris: "{{ data_service_prefix }}/v1/group/activity/agg" upstream_url: "{{ lms_service_url }}/v1/group/activity/agg" From 83b5fb069c700105120f8eebbcc0561e2b8abea1 Mon Sep 17 00:00:00 2001 From: Hari-stackroute <40484996+Hari-stackroute@users.noreply.github.com> Date: Thu, 22 Oct 2020 16:12:33 +0530 Subject: [PATCH 018/184] SB-20938 added feed-delete api details (#2000) * SB-20938 added create and update feed api details * SB-20938 added feed-delete api details --- ansible/roles/kong-api/defaults/main.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 4d533ea44..893415959 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -4360,6 +4360,24 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: userFeedDelete + uris: "{{ user_service_prefix }}/feed/v1/delete" + upstream_url: "{{ learning_service_url }}/v1/user/feed/delete" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'userAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" - name: userFeedUpdate uris: "{{ user_service_prefix }}/feed/v1/update" From af89610a2ac5123e9cd2de1c9adcde2d68e5ca26 Mon Sep 17 00:00:00 2001 From: Hari-stackroute <40484996+Hari-stackroute@users.noreply.github.com> Date: Fri, 23 Oct 2020 12:05:49 +0530 Subject: [PATCH 019/184] SB-21121 limiting feeds to 30 (#2001) * SB-21121 limiting feeds to 30 * SB-21121 limiting feeds to 30 * Update sunbird_learner-service.env update the user-feed property. --- .../roles/stack-sunbird/templates/sunbird_learner-service.env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env index 7c525f229..a03b1c5b4 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env @@ -112,6 +112,8 @@ sunbird_otp_allowed_attempt={{sunbird_otp_allowed_attempt|default(2)}} SUNBIRD_KAFKA_URL={{kafka_urls}} ENV_NAME={{env_name}} notification_service_base_url=http://notification-service:9000 +# Release-3.4.0 +feed_limit={{feed_limit | default(30)}} ## Access Keys ## accesstoken.publickey.basepath={{learner_device_basepath | default('/keys/')}} From 37bccacc9e5218ae6213ada555495ea064e70fe5 Mon Sep 17 00:00:00 2001 From: SMY ALTAMASH <30286162+SMYALTAMASH@users.noreply.github.com> Date: Tue, 27 Oct 2020 17:30:38 +0530 Subject: [PATCH 020/184] Implementing the keycloak URL whitelisting for the entire sunbird stack. (#1995) * Adding all the keycloak endpoints being used by entire sunbird stack * Adding the nginx client IP for x-forwarded-for header and x-real-ip and making remote_addr as default * Remove the mirror configuration * Have a general rule for forgot-password and userdetails admin keycloak --- .../core/nginx-public-ingress/values.j2 | 137 +++++++++--------- 1 file changed, 66 insertions(+), 71 deletions(-) diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 index 9fbb0f7fb..85e92c821 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -116,43 +116,14 @@ proxyconfig: |- proxy_set_header X-Forwarded-Proto $scheme; ignore_invalid_headers off; #pass through headers from Jenkins which are considered invalid by Nginx server. resolver {{ kube_dns_ip }} valid=30s; - location ~* ^/auth/(.*)/impersonation { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/realms/master { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/admin/master/console/ { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/realms/(.+)/token/introspect/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/token/introspect; - } - location ~* ^/auth/realms/(.+)/token/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/token/; - } - location ~* ^/auth/realms/(.+)/userinfo/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/userinfo/; - } - location ~* ^/auth/realms/(.+)/logout/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/logout/; - } - location ~* ^/auth/realms/(.+)/certs/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/certs/; - } - location ~* ^/auth/realms/(.+)/clients-registrations/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/clients-registrations/; - } - location ~* ^/auth/admin/master/console/ { - return 301 {{proto}}://{{ proxy_server_name }}; - } + # Mobile Devices Refresh token Endpoints location ~* ^/auth/v1/refresh/token { rewrite ^/auth/(.*) /auth/$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Forwarded-Proto $scheme; proxy_connect_timeout 5; proxy_send_timeout 60; @@ -161,20 +132,34 @@ proxyconfig: |- proxy_set_header X-Request-ID $sb_request_id; proxy_pass http://kong; } - location /auth/ { + # Admin API Endpoints for sunbird realm fpr forgot password flow + location ~ /auth/admin/realms/sunbird/users/ { rewrite ^/auth/(.*) /auth/$1 break; proxy_set_header X-Request-ID $sb_request_id; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Connection ""; proxy_http_version 1.1; proxy_pass http://keycloak; } - # Caching keycloak static assets - location ~ /auth/resources/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css)) { + # Sunbird realm keycloak API endpoints + location ~ /auth/realms/sunbird/(get-required-action-link|login-actions/(action-token|authenticate|required-action)|protocol/openid-connect/(auth|certs|logout|token|userinfo)|.well-known/openid-configuration) { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Static Assets for keycloak endpoints with caching + location ~ /auth/(resources/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))|welcome-content/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))) { # Enabling caching proxy_cache_key $proxy_host$request_uri; proxy_cache proxy_cache; @@ -190,13 +175,13 @@ proxyconfig: |- proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; - proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_connect_timeout 5; proxy_send_timeout 60; proxy_read_timeout 70; proxy_http_version 1.1; proxy_pass http://keycloak; - } + } # This is Caching mechanism for POST requests location search location ~ /learner/data/v1/location/search { # Enabling caching @@ -984,40 +969,14 @@ keycloakconf: | proxy_set_header X-Forwarded-Proto $scheme; ignore_invalid_headers off; #pass through headers from Jenkins which are considered invalid by Nginx server. resolver 127.0.0.11 valid=5s; - location ~* ^/auth/(.*)/impersonation { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/realms/master { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/admin/master/console/ { - return 301 {{proto}}://{{ proxy_server_name }}; - } - location ~* ^/auth/realms/(.+)/token/introspect/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/token/introspect; - } - location ~* ^/auth/realms/(.+)/token/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/token/; - } - location ~* ^/auth/realms/(.+)/userinfo/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/userinfo/; - } - location ~* ^/auth/realms/(.+)/logout/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/logout/; - } - location ~* ^/auth/realms/(.+)/certs/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/certs/; - } - location ~* ^/auth/realms/(.+)/clients-registrations/ { - return 301 {{proto}}://$host/api/auth/v1/realms/$1/clients-registrations/; - } + # Refresh token endpoint being routed to kong location ~* ^/auth/v1/refresh/token { rewrite ^/auth/(.*) /auth/$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Forwarded-Proto $scheme; proxy_connect_timeout 5; proxy_send_timeout 60; @@ -1026,19 +985,55 @@ keycloakconf: | proxy_set_header X-Request-ID $sb_request_id; proxy_pass http://kong; } - location ~* ^/auth/admin/master/console/ { - return 301 {{proto}}://{{ merge_proxy_server_name }}; + # Admin API Endpoints for sunbird realm fpr forgot password flow + location ~ /auth/admin/realms/sunbird/users/ { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; } - location /auth/ { - set $target {{ keycloak_url }}; + # Sunbird realm keycloak API endpoints + location ~ /auth/realms/sunbird/(get-required-action-link|login-actions/(action-token|authenticate|required-action)|protocol/openid-connect/(auth|certs|logout|token|userinfo)|.well-known/openid-configuration) { rewrite ^/auth/(.*) /auth/$1 break; proxy_set_header X-Request-ID $sb_request_id; - proxy_pass $target; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Static Assets for keycloak endpoints with caching + location ~ /auth/(resources/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))|welcome-content/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))) { + # Enabling caching + proxy_cache_key $proxy_host$request_uri; + proxy_cache proxy_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 14400; + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://keycloak; } location / { rewrite ^/(.*) /$1 break; From 244400778e0296b860d6141d2da1b01445fbbb76 Mon Sep 17 00:00:00 2001 From: Rhea Fernandes <32357913+rheafernandes@users.noreply.github.com> Date: Tue, 27 Oct 2020 19:14:08 +0530 Subject: [PATCH 021/184] Added code changes for forms api support (#2006) --- .../stack-sunbird/templates/content-service_application.conf | 1 + .../stack-sunbird/templates/taxonomy-service_application.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf index 12f649202..e5d252180 100644 --- a/ansible/roles/stack-sunbird/templates/content-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf @@ -564,4 +564,5 @@ mimeTypeToPrimaryCategory { channel.content.primarycategories=["Explanation Content", "Learning Resource", "Practice Question Set", "eTextbook", "Teacher Resource", "Course Assessment"] channel.collection.primarycategories=["Course", "Digital Textbook", "Content Playlist"] channel.asset.primarycategories=["Asset"] +objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" diff --git a/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf b/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf index fe251b214..66a27cfe6 100644 --- a/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf @@ -389,3 +389,4 @@ languageCode { tamil : "ta" telugu : "te" } +objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" From fc8845be2f8487412710209cb864ba96b054460f Mon Sep 17 00:00:00 2001 From: sknirmalkar89 <59080585+sknirmalkar89@users.noreply.github.com> Date: Wed, 28 Oct 2020 17:37:43 +0530 Subject: [PATCH 022/184] SB-20957 Create a new jenkins scripts for groups data correction (#2007) * SB-20957 new jenkins job for groups service * SB-20957 adding jenkins to run paramter based jar * SB-20957 adding jenkins to run paramter based jar --- ansible/cassandra-deploy-decrypt.yml | 1 + ansible/roles/cassandra-deploy-decrypt/defaults/main.yml | 1 + ansible/roles/cassandra-deploy-decrypt/tasks/main.yml | 4 ++-- pipelines/deploy/cassandra-deploy/Jenkinsfile.decrypt | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/cassandra-deploy-decrypt.yml b/ansible/cassandra-deploy-decrypt.yml index 341631a4b..fa0053cdc 100644 --- a/ansible/cassandra-deploy-decrypt.yml +++ b/ansible/cassandra-deploy-decrypt.yml @@ -5,6 +5,7 @@ sunbird_cassandra_port: 9042 sunbird_cassandra_keyspace: sunbird sunbird_encryption_key: "{{ core_vault_sunbird_encryption_key }}" + sunbird_cassandra_groups_keyspace: sunbird_groups vars_files: - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] roles: diff --git a/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml b/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml index c38c2c9d3..02efebd0e 100644 --- a/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml +++ b/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml @@ -1,2 +1,3 @@ cassandra_jar_path: ../ cassandra_deploy_path: /home/deployer +cassandra_jar_name: user-declaration-util-1.0-SNAPSHOT-jar-with-dependencies.jar diff --git a/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml b/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml index 54787a84e..0275452cd 100644 --- a/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml +++ b/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml @@ -1,10 +1,10 @@ - name: Copy the jar become: yes copy: - src: "{{cassandra_jar_path}}/user-declaration-util-1.0-SNAPSHOT-jar-with-dependencies.jar" + src: "{{cassandra_jar_path}}/{{cassandra_jar_name}}" dest: "{{cassandra_deploy_path}}" run_once: true - name: Run the application - shell: java -jar user-declaration-util-1.0-SNAPSHOT-jar-with-dependencies.jar + shell: "java -jar {{cassandra_jar_name}}" run_once: true diff --git a/pipelines/deploy/cassandra-deploy/Jenkinsfile.decrypt b/pipelines/deploy/cassandra-deploy/Jenkinsfile.decrypt index c695d7a97..ae6eae690 100644 --- a/pipelines/deploy/cassandra-deploy/Jenkinsfile.decrypt +++ b/pipelines/deploy/cassandra-deploy/Jenkinsfile.decrypt @@ -30,7 +30,7 @@ node() { unzip ${artifact} """ ansiblePlaybook = "${currentWs}/ansible/cassandra-deploy-decrypt.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + ansibleExtraArgs = "-e cassandra_jar_name=\"${params.cassandra_jar_name}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v " values.put('ansiblePlaybook', ansiblePlaybook) values.put('ansibleExtraArgs', ansibleExtraArgs) println values From 8e63180028a3655794f5e31ca1dbe1ec3340659a Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 2 Nov 2020 12:20:36 +0530 Subject: [PATCH 023/184] Merge 3.3.0 to 3.4.0 (#2015) * Issue #SB-21135 feat: Updated LMS configuration (#2003) * Issue #SC-1929 feat: Courses logback changes * Issue #SC-2041 feat: Updated issue certificate topic config * Issue #SB-21135 feat: Updated LMS configuration * Issue #SB-21135 feat: Updated LMS configuration * Issue #SB-21135 feat: Updated LMS configuration * Enanbling secured way to upload files using jenkins (#2004) * Issue #SB-21123 fix: updated config for search-service (#2005) * Issue #SB-21123 fix: updated config for search-service * Issue #SB-21123 fix: updated variable name * Merge 3.2.7 to 3.3.0 (#2008) * Added the secor process alert rules. (#2012) * added the druid alert rules (#1996) * Issue #TG607: added the druid alert rules * Issue #TG607: added the druid alert rules * Issue #TG607: updated the druid alert rules * Issue #0000: added the datasource name in message field. * Issue #0000: removed the rollup datasource * Issue #0000: added the secor process alert rules * Issue #0000: added the email config * Issue #0000: added the secor processes names * Issue #0000: updated the alert name * Issue #0000: removed the old secor alertrules * Issue #TG-641: updated the alert rules Co-authored-by: Pradyumna Co-authored-by: SMY ALTAMASH <30286162+SMYALTAMASH@users.noreply.github.com> Co-authored-by: Kumar Gauraw Co-authored-by: kaliraja <34502260+Kaali09@users.noreply.github.com> --- ansible/cert-file-upload.yml | 10 ++ .../roles/cert-file-upload/defaults/main.yml | 2 + ansible/roles/cert-file-upload/tasks/main.yml | 7 ++ ansible/roles/kong-api/defaults/main.yml | 1 - .../content-service_application.conf | 1 - .../sunbird_knowledge-mw-service.env | 4 +- .../templates/sunbird_lms-service.env | 4 +- .../roles/sunbird-deploy/tasks/main.yml | 2 +- .../templates/alertrules.yaml | 20 ++++ .../templates/prometheus-operator.yaml | 23 +++- .../templates/daemonset.yaml | 2 + .../core/nginx-public-ingress/values.j2 | 106 ++++++++++++++++-- .../alertrules/templates/promrulesProcess.yml | 24 ---- .../templates/promrulesSecorProcess.yml | 29 +++++ .../dashboards/dashboards/nginx-detailed.json | 38 +++---- pipelines/certs-templates/Jenkinsfile.upload | 61 ++++++++++ 16 files changed, 268 insertions(+), 66 deletions(-) create mode 100644 ansible/cert-file-upload.yml create mode 100644 ansible/roles/cert-file-upload/defaults/main.yml create mode 100644 ansible/roles/cert-file-upload/tasks/main.yml create mode 100644 kubernetes/helm_charts/monitoring/alertrules/templates/promrulesSecorProcess.yml create mode 100644 pipelines/certs-templates/Jenkinsfile.upload diff --git a/ansible/cert-file-upload.yml b/ansible/cert-file-upload.yml new file mode 100644 index 000000000..e29a7b6c2 --- /dev/null +++ b/ansible/cert-file-upload.yml @@ -0,0 +1,10 @@ +--- +- hosts: localhost + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_public_storage_account_key }}" + roles: + - cert-file-upload diff --git a/ansible/roles/cert-file-upload/defaults/main.yml b/ansible/roles/cert-file-upload/defaults/main.yml new file mode 100644 index 000000000..b74bef14c --- /dev/null +++ b/ansible/roles/cert-file-upload/defaults/main.yml @@ -0,0 +1,2 @@ +file_name: "certUploadedfile" +cert_template_artifacts_container_name: "e-credentials" diff --git a/ansible/roles/cert-file-upload/tasks/main.yml b/ansible/roles/cert-file-upload/tasks/main.yml new file mode 100644 index 000000000..9fcaf6a67 --- /dev/null +++ b/ansible/roles/cert-file-upload/tasks/main.yml @@ -0,0 +1,7 @@ +- name: rename the file to the correct name as required in blobstore + command: "mv /tmp/certUploadedfile /tmp/{{file_name}}" + +- name: Upload to azure blob storage + command: "az storage blob upload -c {{ cert_template_artifacts_container_name }} --name {{ file_name }} -f /tmp/{{file_name}}" + async: 3600 + poll: 10 diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 893415959..1c01c239f 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -1,4 +1,3 @@ - --- # Guideline for PR Merging: https://project-sunbird.atlassian.net/wiki/spaces/DevOps/pages/1210777612/API+Manager+Kong+BEST+Practices+Documentation. diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf index e5d252180..fb31f84f4 100644 --- a/ansible/roles/stack-sunbird/templates/content-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf @@ -565,4 +565,3 @@ channel.content.primarycategories=["Explanation Content", "Learning Resource", " channel.collection.primarycategories=["Course", "Digital Textbook", "Content Playlist"] channel.asset.primarycategories=["Asset"] objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" - diff --git a/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env b/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env index 6a17b9f2d..ec8132a2a 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env @@ -19,11 +19,11 @@ sunbird_content_service_blacklisted_channels={{content_service_blacklisted_chann sunbird_default_channel={{sunbird_default_channel}} sunbird_content_repo_api_base_url={{sunbird_content_repo_api_base_url}} sunbird_content_repo_api_key={{sunbird_content_repo_api_key}} -sunbird_search_service_api_base_url={{sunbird_search_service_api_base_url}} +sunbird_search_service_api_base_url={{knowledge_mw_sunbird_search_service_api_base_url | default('http://search-service:9000')}} sunbird_search_service_api_key={{sunbird_search_service_api_key}} sunbird_dial_repo_api_base_url={{sunbird_dial_repo_api_base_url}} sunbird_dial_repo_api_key={{sunbird_dial_repo_api_key}} -sunbird_plugin_repo_api_base_url={{sunbird_plugin_repo_api_base_url}} +sunbird_plugin_repo_api_base_url={{knowledge_mw_sunbird_plugin_repo_api_base_url | default('http://search-service:9000')}} sunbird_plugin_repo_api_key={{sunbird_plugin_repo_api_key}} sunbird_data_service_api_base_url={{sunbird_data_service_api_base_url}} sunbird_data_service_api_key={{sunbird_data_service_api_key}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index 958982df0..42859c390 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -19,8 +19,8 @@ sunbird_pg_user={{sunbird_pg_user}} sunbird_pg_password={{sunbird_pg_password}} sunbird_installation={{sunbird_installation}} sunbird_analytics_api_base_url={{sunbird_analytics_api_base_url}} -sunbird_search_service_api_base_url={{sunbird_search_service_api_base_url}} -ekstep_api_base_url={{sunbird_content_service_api_base_url}} +sunbird_search_service_api_base_url={{ lms_search_service_api_base_url | default('http://search-service:9000') }} +ekstep_api_base_url={{ lms_ekstep_api_base_url | default('http://content-service:9000') }} sunbird_group_service_api_base_url={{sunbird_group_service_api_base_url}} sunbird_mail_server_host={{sunbird_mail_server_host}} sunbird_mail_server_port={{sunbird_mail_server_port}} diff --git a/kubernetes/ansible/roles/sunbird-deploy/tasks/main.yml b/kubernetes/ansible/roles/sunbird-deploy/tasks/main.yml index 95b651e36..9d7ad2ee9 100644 --- a/kubernetes/ansible/roles/sunbird-deploy/tasks/main.yml +++ b/kubernetes/ansible/roles/sunbird-deploy/tasks/main.yml @@ -41,7 +41,7 @@ when: release_name == "learner" or release_name == "lms" or release_name == "groups" - name: helm upgrade - shell: helm upgrade --install --atomic --timeout 10m {{ release_name }} {{ chart_path }} -n {{namespace}} + shell: helm upgrade --install --atomic --timeout {{helm_install_timeout | d('10m')}} {{ release_name }} {{ chart_path }} -n {{namespace}} - name: Clean up keys include_role: diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml index 5a6466670..b448efd7d 100644 --- a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml @@ -59,3 +59,23 @@ raw_druid_datasource: druid_raw_ingestion_threshold: 200000 druid_rollup_ingestion_threshold: 200000 +secor_process_names: + - channel-telemetry-backup + - derived-telemetry-backup + - derived-denorm-events-backup + - channel-summary-backup + - assess-events-backup + - assess-raw-events-backup + - device-profile-backup + - learning-events-backup + - learning-failed-backup + - content-consumption-events-backup + - failed-telemetry-backup + - extractor-duplicate-backup + - extractor-failed-backup + - unique-telemetry-backup + - denorm-events-backup + - raw-telemetry-backup + - ingestion-telemetry-backup + - ingestion-cluster-telemetry-backup + - telemetry-ingest-backup diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml index 11d250f2e..ce05bde3a 100644 --- a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml @@ -145,11 +145,18 @@ alertmanager: text: |- {{ "{{" }} range .Alerts {{ "}}" }} *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} - *processName:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *ProcessName:* {{ "{{" }} .Annotations.job_id {{ "}}" }} *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} {{ "{{" }} end {{ "}}" }} icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' - name: 'dp-lag_slack_warning' slack_configs: @@ -168,6 +175,13 @@ alertmanager: *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} {{ "{{" }} end {{ "}}" }} icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' - name: 'dp-lag_slack_critical' slack_configs: @@ -186,6 +200,13 @@ alertmanager: *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} {{ "{{" }} end {{ "}}" }} icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' {% for item in alert_teams %} # Comment to ensure proper indentation while templating diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml index 929a28e6c..3b647c055 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml @@ -10,6 +10,8 @@ spec: app: nginx-public-ingress updateStrategy: type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% template: metadata: annotations: diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 index 85e92c821..b10380df3 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -169,7 +169,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; rewrite ^/auth/(.*) /auth/$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; @@ -194,7 +194,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; @@ -219,7 +219,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 3600s; + proxy_cache_valid 200 14400s; # Increasing the proxy buffer size proxy_buffer_size 16k; proxy_busy_buffers_size 16k; @@ -246,7 +246,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 3600s; + proxy_cache_valid 200 14400s; # Increasing the proxy buffer size proxy_buffer_size 16k; proxy_busy_buffers_size 16k; @@ -273,7 +273,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; rewrite ^/api/(.*) /$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; @@ -297,7 +297,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; rewrite ^/api/(.*) /$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; @@ -705,6 +705,90 @@ proxyconfig: |- proxy_set_header X-Request-ID $sb_request_id; proxy_pass https://$offline_bucket/$url_full; } + # compression for svg certs download + location /api/certreg/v2/certs/download { + rewrite ^/api/(.*) /$1 break; + # Compression + gzip on; + gzip_comp_level 5; + gzip_min_length 50000; # 50KB + gzip_proxied any; + gzip_vary on; + # Content types for compression + gzip_types + application/atom+xml + application/javascript + application/json + application/ld+json + application/manifest+json + application/rss+xml + application/vnd.geo+json + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/bmp + image/svg+xml + image/x-icon + text/cache-manifest + text/css + text/plain + add_header test hello; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + location /learner/certreg/v2/certs/download { + # Compression + gzip on; + gzip_comp_level 5; + gzip_min_length 50000; # 50KB + gzip_proxied any; + gzip_vary on; + # Content types for compression + gzip_types + application/atom+xml + application/javascript + application/json + application/ld+json + application/manifest+json + application/rss+xml + application/vnd.geo+json + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/bmp + image/svg+xml + image/x-icon + text/cache-manifest + text/css + text/plain + add_header test hello; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } location / { rewrite ^/(.*) /$1 break; proxy_set_header Host $host; @@ -747,14 +831,14 @@ proxyconfig: |- } location ~ /resourcebundles/v1/read|/learner/data/v1/(role/read|system/settings/get)|/v1/tenant/info { proxy_cache_key $proxy_host$request_uri; - proxy_cache proxy_cache; + proxy_cache framework_cache; add_header X-Proxy-Cache $upstream_cache_status; add_header X-Proxy-Cache-Date $upstream_http_date; proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; proxy_set_header Host $host; proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; proxy_set_header X-Scheme $scheme; @@ -772,7 +856,7 @@ proxyconfig: |- proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_valid 200 14400; + proxy_cache_valid 200 43200; rewrite ^/api/channel/v1/read/(.*) /channel/v1/read/$1 break; proxy_set_header Connection ""; proxy_set_header Host $host; @@ -925,11 +1009,11 @@ nginxconfig: | # caching for images and files proxy_cache_path /tmp/proxy_cache levels=1:2 keys_zone=tmp_cache:5m max_size=10m inactive=300m use_temp_path=off; # caching for apis - proxy_cache_path /tmp/api_cache levels=1:2 keys_zone=proxy_cache:5m max_size=300m inactive=300m use_temp_path=off; + proxy_cache_path /tmp/api_cache levels=1:2 keys_zone=proxy_cache:5m max_size=600m inactive=1400m use_temp_path=off; # cache framework proxy_cache_path /tmp/framework_cache levels=1:2 keys_zone=framework_cache:5m max_size=700m inactive=300m use_temp_path=off; # cache content - proxy_cache_path /tmp/content_cache levels=1:2 keys_zone=content_cache:100m max_size=1500m inactive=300m use_temp_path=off; + proxy_cache_path /tmp/content_cache levels=1:2 keys_zone=content_cache:100m max_size=3000m inactive=600m use_temp_path=off; # cache content metadata proxy_cache_path /tmp/content_metadata levels=1:2 keys_zone=content_metadata:100m max_size=1000m inactive=300m use_temp_path=off; diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesProcess.yml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesProcess.yml index cd2c2a701..39c76ee1a 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesProcess.yml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesProcess.yml @@ -60,30 +60,6 @@ spec: job_id: kafka alertname: TooManyKafkaProcessRunning - - alert: secor_process_not_running_critical - expr: namedprocess_namegroup_states{groupname="secor",state="Sleeping"} != {{ .Values.secor_job_count }} - for: 1m - labels: - severity: critical - module: dp_process - annotations: - message: Number of running processes should be {{ .Values.secor_job_count }} but currently {{`{{$value}}`}} processes are running. - summary: Secor process is not running - job_id: secor - alertname: SecorJobNotRunning - - - alert: secor_process_not_running_critical - expr: namedprocess_namegroup_states{groupname="processingsecor",state="Sleeping"} != {{ .Values.processingsecor_job_count }} - for: 1m - labels: - severity: critical - module: dp_process - annotations: - message: Number of running processes should be {{ .Values.secor_job_count }} but currently {{`{{$value}}`}} processes are running. - summary: Secor process is not running - job_id: secor - alertname: SecorJobNotRunning - - alert: zookeeper_process_not_running_fatal expr: namedprocess_namegroup_states{groupname="zookeeper",state="Sleeping"} < 1 for: 1m diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesSecorProcess.yml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesSecorProcess.yml new file mode 100644 index 000000000..0343e106e --- /dev/null +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesSecorProcess.yml @@ -0,0 +1,29 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + role: alert-rules + app: {{ .Values.prometheus_rule_selector_app }} + release: {{ .Values.prometheus_rule_selector_release }} + name: {{ .Values.fullnameOverride }}-secor-process-rules + namespace: {{ .Values.namespace }} +spec: + groups: + - name: alertrules.process + rules: + {{- if .Values.secor_process_names }} + ## druid segment unavailable count alert + {{ range $key := .Values.secor_process_names }} + - alert: secor_process_not_running_critical {{ . }} + expr: sum without(state) (namedprocess_namegroup_states{groupname={{ . | quote }}}) == 0 + for: 5m + labels: + severity: critical + module: dp_process + annotations: + message: The secor job {{ . }} is not running. + job_id: {{ . }} + alertname: SecorJobNotRunning + {{- end }} + {{- end }} diff --git a/kubernetes/helm_charts/monitoring/dashboards/dashboards/nginx-detailed.json b/kubernetes/helm_charts/monitoring/dashboards/dashboards/nginx-detailed.json index fbaea070b..a5bcfeb7c 100644 --- a/kubernetes/helm_charts/monitoring/dashboards/dashboards/nginx-detailed.json +++ b/kubernetes/helm_charts/monitoring/dashboards/dashboards/nginx-detailed.json @@ -17,7 +17,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1601270090088, + "iteration": 1603867463241, "links": [], "panels": [ { @@ -188,19 +188,19 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] offset 1d)) / sum(increase(nginx_http_requests_total[5m] ))) * 100", + "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] )) / sum(increase(nginx_http_requests_total[5m] ))) * 100", "interval": "", "legendFormat": "Today", "refId": "A" }, { - "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] offset 1d)) / sum(increase(nginx_http_requests_total[5m] offset 1w))) * 100", + "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] offset 1d)) / sum(increase(nginx_http_requests_total[5m] offset 1d))) * 100", "interval": "", "legendFormat": "Yesteday", "refId": "B" }, { - "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] offset 1d)) / sum(increase(nginx_http_requests_total[5m] offset 1d))) * 100", + "expr": "(sum(increase(nginx_http_requests_total{status!~\"4..|5..\"}[5m] offset 1w)) / sum(increase(nginx_http_requests_total[5m] offset 1w))) * 100", "interval": "", "legendFormat": "Last Week", "refId": "C" @@ -296,13 +296,13 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(increase(nginx_http_requests_total{ status!~\"2..|3..\"}[5m] offset 1w)) / sum(increase(nginx_http_requests_total[5m]))) * 100", + "expr": "(sum(increase(nginx_http_requests_total{ status!~\"2..|3..\"}[5m])) / sum(increase(nginx_http_requests_total[5m]))) * 100", "interval": "", "legendFormat": "Today", "refId": "A" }, { - "expr": "(sum(increase(nginx_http_requests_total{ status!~\"2..|3..\"}[5m] offset 1w)) / sum(increase(nginx_http_requests_total[5m] offset 1d))) * 100", + "expr": "(sum(increase(nginx_http_requests_total{ status!~\"2..|3..\"}[5m] offset 1d)) / sum(increase(nginx_http_requests_total[5m] offset 1d))) * 100", "interval": "", "legendFormat": "Yesterday", "refId": "B" @@ -451,7 +451,7 @@ "step": 2 }, { - "expr": "sum(rate(nginx_http_requests_total{cluster=~\"$cluster\", status=\"444\"}[5m]))", + "expr": "sum(rate(nginx_http_requests_total{cluster=~\"$cluster\", status=\"345\"}[5m]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1062,13 +1062,11 @@ "legend": { "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -1088,23 +1086,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(nginx_http_connections) by (instance)", + "expr": "sum(nginx_http_connections) by (state)", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{ instance }}", "refId": "A" - }, - { - "expr": "sum(nginx_http_connections)", - "interval": "", - "legendFormat": "Total", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Nginx Total Connection", + "title": "Nginx Connections", "tooltip": { "shared": true, "sort": 0, @@ -1120,7 +1112,7 @@ }, "yaxes": [ { - "$$hashKey": "object:1799", + "$$hashKey": "object:142", "format": "short", "label": null, "logBase": 1, @@ -1129,7 +1121,7 @@ "show": true }, { - "$$hashKey": "object:1800", + "$$hashKey": "object:143", "format": "short", "label": null, "logBase": 1, @@ -1240,7 +1232,7 @@ } } ], - "refresh": "1m", + "refresh": "5m", "schemaVersion": 25, "style": "dark", "tags": [], @@ -1305,5 +1297,5 @@ "timezone": "", "title": "Nginx Detailed", "uid": "9wKzPxZMz", - "version": 1 + "version": 2 } diff --git a/pipelines/certs-templates/Jenkinsfile.upload b/pipelines/certs-templates/Jenkinsfile.upload new file mode 100644 index 000000000..8b0ef0a6b --- /dev/null +++ b/pipelines/certs-templates/Jenkinsfile.upload @@ -0,0 +1,61 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage("upload") { + def inputFile = input message: 'Upload file', parameters: [file(name: 'certUploadedfile')] + new hudson.FilePath(new File("tmp/certUploadedfile")).copyFrom(inputFile) + } + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/cert-file-upload.yml" + ansibleExtraArgs = "--extra-vars \"file_name=${params.file_name}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + + stage('remove tmp file'){ + sh """ + rm -rf /tmp/certUploadedfile + """ + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} From bac360bd5abf1bc6cd7180839eca312601ce6d41 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 4 Nov 2020 12:58:06 +0530 Subject: [PATCH 024/184] Adding fluent-bit chart (#2019) Adding type conversion for nginx logs --- .../roles/logging/templates/fluent-bit.yaml | 27 +- .../logging/fluent-bit/.helmignore | 23 + .../helm_charts/logging/fluent-bit/Chart.yaml | 42 +- .../helm_charts/logging/fluent-bit/OWNERS | 10 - .../helm_charts/logging/fluent-bit/README.md | 191 --- .../logging/fluent-bit/templates/NOTES.txt | 15 - .../logging/fluent-bit/templates/_helpers.tpl | 90 +- .../fluent-bit/templates/cluster-role.yaml | 28 - .../templates/cluster-rolebinding.yaml | 19 - .../logging/fluent-bit/templates/config.yaml | 271 --- .../fluent-bit/templates/configMap.yaml | 166 ++ .../fluent-bit/templates/daemonset.yaml | 208 +-- .../fluent-bit/templates/endpoint.yaml | 27 + .../logging/fluent-bit/templates/psp.yaml | 60 - .../logging/fluent-bit/templates/secret.yaml | 30 - .../logging/fluent-bit/templates/service.yaml | 34 +- .../fluent-bit/templates/serviceMonitor.yaml | 27 + .../fluent-bit/templates/serviceaccount.yaml | 38 +- .../fluent-bit/templates/servicemonitor.yaml | 34 - .../templates/tests/test-configmap.yaml | 48 - .../fluent-bit/templates/tests/test.yaml | 53 - .../logging/fluent-bit/values.yaml | 354 +--- .../dashboards/dashboards/fluent-bit.json | 1509 +++++++++++++++++ .../monitoring/dashboards/values.yaml | 3 + 24 files changed, 1973 insertions(+), 1334 deletions(-) create mode 100644 kubernetes/helm_charts/logging/fluent-bit/.helmignore mode change 100755 => 100644 kubernetes/helm_charts/logging/fluent-bit/Chart.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/OWNERS delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/README.md delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/NOTES.txt delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/cluster-role.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/cluster-rolebinding.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/config.yaml create mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml create mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/endpoint.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/psp.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/secret.yaml create mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/serviceMonitor.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/servicemonitor.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/tests/test-configmap.yaml delete mode 100644 kubernetes/helm_charts/logging/fluent-bit/templates/tests/test.yaml create mode 100644 kubernetes/helm_charts/monitoring/dashboards/dashboards/fluent-bit.json diff --git a/kubernetes/ansible/roles/logging/templates/fluent-bit.yaml b/kubernetes/ansible/roles/logging/templates/fluent-bit.yaml index f7bce5c1b..1d85dc620 100644 --- a/kubernetes/ansible/roles/logging/templates/fluent-bit.yaml +++ b/kubernetes/ansible/roles/logging/templates/fluent-bit.yaml @@ -1,24 +1,3 @@ -backend: - type: es - es: - host: {{ logger_es6_host }} - port: {{ logger_es_port }} - index: logstash-fb - type: flb_type - logstash_prefix: logstash-fb - replace_dots: "On" - logstash_format: "On" - retry_limit: "False" - time_key: "@timestamp" - # Optional username credential for Elastic X-Pack access - http_user: - # Password for user defined in HTTP_User - http_passwd: - # Optional TLS encryption to ElasticSearch instance - tls: "off" - tls_verify: "on" - # TLS certificate for the Elastic (in PEM format). Use if tls=on and tls_verify=on. - tls_ca: "" - # TLS debugging levels = 1-4 - tls_debug: 1 - +es: + endpoint: true + ips: {{groups['log-es'] | to_json}} diff --git a/kubernetes/helm_charts/logging/fluent-bit/.helmignore b/kubernetes/helm_charts/logging/fluent-bit/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml b/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml old mode 100755 new mode 100644 index 2aaf9640d..47ff4d0df --- a/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml +++ b/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml @@ -1,21 +1,23 @@ -apiVersion: v1 +apiVersion: v2 name: fluent-bit -version: 2.8.12 -appVersion: 1.3.7 -description: Fast and Lightweight Log/Data Forwarder for Linux, BSD and OSX -keywords: -- logging -- monitoring -- fluent -- fluentd -sources: -- https://fluentbit.io -icon: https://fluentbit.io/assets/img/logo1-default.png -home: https://fluentbit.io -maintainers: -- name: kfox1111 - email: Kevin.Fox@pnnl.gov -- name: edsiper - email: eduardo@treasure-data.com -- name: hectorj2f - email: hfernandez@mesosphere.com +description: Fluentbit logging helm + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.6.3 diff --git a/kubernetes/helm_charts/logging/fluent-bit/OWNERS b/kubernetes/helm_charts/logging/fluent-bit/OWNERS deleted file mode 100644 index fa1f0128d..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/OWNERS +++ /dev/null @@ -1,10 +0,0 @@ -approvers: -- kfox1111 -- edsiper -- hectorj2f -- Towmeykaw -reviewers: -- kfox1111 -- edsiper -- hectorj2f -- Towmeykaw diff --git a/kubernetes/helm_charts/logging/fluent-bit/README.md b/kubernetes/helm_charts/logging/fluent-bit/README.md deleted file mode 100644 index 2ece5deed..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/README.md +++ /dev/null @@ -1,191 +0,0 @@ -# Fluent-Bit Chart - -[Fluent Bit](http://fluentbit.io/) is an open source and multi-platform Log Forwarder. - -## Chart Details - -This chart will do the following: - -* Install a configmap for Fluent Bit -* Install a daemonset that provisions Fluent Bit [per-host architecture] - -## Installing the Chart - -To install the chart with the release name `my-release`: - -```bash -$ helm install --name my-release stable/fluent-bit -``` - -When installing this chart on [Minikube](https://kubernetes.io/docs/getting-started-guides/minikube/), it's required to specify that so the DaemonSet will be able to mount the log files properly, make sure to append the _--set on\_minikube=true_ option at the end of the _helm_ command, e.g: - -```bash -$ helm install --name my-release stable/fluent-bit --set on_minikube=true -``` - -## Configuration - -The following table lists the configurable parameters of the Fluent-Bit chart and the default values. - -| Parameter | Description | Default | -| ----------------------- | ---------------------------------- | ----------------------- | -| **Backend Selection** | -| `backend.type` | Set the backend to which Fluent-Bit should flush the information it gathers | `forward` | -| **Forward Backend** | -| `backend.forward.host` | Target host where Fluent-Bit or Fluentd are listening for Forward messages | `fluentd` | -| `backend.forward.port` | TCP Port of the target service | `24284` | -| `backend.forward.shared_key` | A key string known by the remote Fluentd used for authorization. | `` | -| `backend.forward.tls` | Enable or disable TLS support | `off` | -| `backend.forward.tls_verify` | Force certificate validation | `on` | -| `backend.forward.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | -| **ElasticSearch Backend** | -| `backend.es.host` | IP address or hostname of the target Elasticsearch instance | `elasticsearch` | -| `backend.es.port` | TCP port of the target Elasticsearch instance. | `9200` | -| `backend.es.index` | Elastic Index name | `kubernetes_cluster` | -| `backend.es.type` | Elastic Type name | `flb_type` | -| `backend.es.retry_limit` | Max number of retries to attempt (False == no limit) | `False` | -| `backend.es.time_key` | Elastic Time Key | `@timestamp` | -| `backend.es.logstash_format` | Enable Logstash format compatibility. | `On` | -| `backend.es.logstash_prefix` | Index Prefix. If Logstash_Prefix is equal to 'mydata' your index will become 'mydata-YYYY.MM.DD'. | `kubernetes_cluster` | -| `backend.es.logstash_prefix_key` | Index Prefix key. When included, the value in the record that belongs to the key will be looked up and overwrite `Logstash_Prefix` for index generation. If `Logstash_Prefix_Key` = 'mydata' the index becomes 'mydata-YYYY.MM.DD'. | `` | -| `backend.es.replace_dots` | Enable/Disable Replace_Dots option. | `On` | -| `backend.es.http_user` | Optional username credential for Elastic X-Pack access. | `` | -| `backend.es.http_passwd` | Password for user defined in HTTP_User. | `` | -| `backend.es.http_passwd_secret` | Secret name for password for user defined in HTTP_User. | `` | -| `backend.es.http_passwd_secret_key` | Secret key for password for user defined in HTTP_User. | `` | -| `backend.es.tls` | Enable or disable TLS support | `off` | -| `backend.es.tls_verify` | Force certificate validation | `on` | -| `backend.es.tls_secret` | Existing secret storing TLS CA certificate for the Elastic instance. Specify if tls: on. Overrides `backend.es.tls_ca` | `` | -| `backend.es.tls_secret_ca_key` | Existing secret key storing TLS CA certificate for the Elastic instance. Specify if tls: on. | `` | -| `backend.es.tls_ca` | TLS CA certificate for the Elastic instance (in PEM format). Specify if tls: on. | `` | -| `backend.es.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | -| **HTTP Backend** | -| `backend.http.host` | IP address or hostname of the target HTTP Server | `127.0.0.1` | -| `backend.http.port` | TCP port of the target HTTP Server | `80` | -| `backend.http.uri` | Specify an optional HTTP URI for the target web server, e.g: /something | `"/"` -| `backend.http.http_user` | Optional username credential for Basic Authentication. | `` | -| `backend.http.http_passwd:` | Password for user defined in HTTP_User. | `` | -| `backend.http.format` | Specify the data format to be used in the HTTP request body, by default it uses msgpack, optionally it can be set to json. | `msgpack` | -| `backend.http.json_date_format` | Specify the format of the date. Supported formats are double and iso8601 | `double` | -| `backend.http.headers` | HTTP Headers | `[]` | -| `backend.http.tls` | Enable or disable TLS support | `off` | -| `backend.http.tls_verify` | Force certificate validation | `on` | -| `backend.http.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | -| **Splunk Backend** | -| `backend.splunk.host` | IP address or hostname of the target Splunk Server | `127.0.0.1` | -| `backend.splunk.port` | TCP port of the target Splunk Server | `8088` | -| `backend.splunk.token` | Specify the Authentication Token for the HTTP Event Collector interface. | `` | -| `backend.splunk.send_raw` | If enabled, record keys and values are set in the main map. | `off` | -| `backend.splunk.tls` | Enable or disable TLS support | `on` | -| `backend.splunk.tls_verify` | Force TLS certificate validation | `off` | -| `backend.splunk.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | -| `backend.splunk.message_key` | Tag applied to all incoming logs | `kubernetes` | -| **Stackdriver Backend** | -| `backend.stackdriver.google_service_credentials` | Contents of a Google Cloud credentials JSON file. | `` | -| `backend.stackdriver.service_account_email` | Account email associated to the service. Only available if no credentials file has been provided. | `` | -| `backend.stackdriver.service_account_secret` | Private key content associated with the service account. Only available if no credentials file has been provided. | `` | -| **Parsers** | -| `parsers.enabled` | Enable custom parsers | `false` | -| `parsers.regex` | List of regex parsers | `NULL` | -| `parsers.json` | List of json parsers | `NULL` | -| `parsers.logfmt` | List of logfmt parsers | `NULL` | -| **General** | -| `annotations` | Optional deamonset set annotations | `NULL` | -| `audit.enable` | Enable collection of audit logs | `false` | -| `audit.input.memBufLimit` | Specify Mem_Buf_Limit in tail input | `35mb` | -| `audit.input.parser` | Specify Parser in tail input | `docker` | -| `audit.input.tag` | Specify Tag in tail input | `audit.*` | -| `audit.input.path` | Specify log file(s) through the use of common wildcards | `/var/log/kube-apiserver-audit.log` | -| `audit.input.bufferChunkSize` | Specify Buffer_Chunk_Size in tail | `2MB` | -| `audit.input.bufferMaxSize` | Specify Buffer_Max_Size in tail | `10MB` | -| `audit.input.skipLongLines` | Specify Skip_Long_Lines in tail | `On` | -| `audit.input.key` | Specify Key in tail | `kubernetes-audit` | -| `podAnnotations` | Optional pod annotations | `NULL` | -| `podLabels` | Optional pod labels | `NULL` | -| `fullConfigMap` | User has provided entire config (parsers + system) | `false` | -| `existingConfigMap` | ConfigMap override | `` | -| `extraEntries.input` | Extra entries for existing [INPUT] section | `` | -| `extraEntries.filter` | Extra entries for existing [FILTER] section | `` | -| `extraEntries.output` | Extra entries for existing [OUPUT] section | `` | -| `extraPorts` | List of extra ports | | -| `extraVolumeMounts` | Mount an extra volume, required to mount ssl certificates when elasticsearch has tls enabled | | -| `extraVolume` | Extra volume | | -| `service.flush` | Interval to flush output (seconds) | `1` | -| `service.logLevel` | Diagnostic level (error/warning/info/debug/trace) | `info` | -| `filter.enableExclude` | Enable the use of monitoring for a pod annotation of `fluentbit.io/exclude: true`. If present, discard logs from that pod. | `true` | -| `filter.enableParser` | Enable the use of monitoring for a pod annotation of `fluentbit.io/parser: parser_name`. parser_name must be the name of a parser contained within parsers.conf | `true` | -| `filter.kubeURL` | Optional custom configmaps | `https://kubernetes.default.svc:443` | -| `filter.kubeCAFile` | Optional custom configmaps | `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` | -| `filter.kubeTokenFile` | Optional custom configmaps | `/var/run/secrets/kubernetes.io/serviceaccount/token` | -| `filter.kubeTag` | Optional top-level tag for matching in filter | `kube` | -| `filter.kubeTagPrefix` | Optional tag prefix used by Tail | `kube.var.log.containers.` | -| `filter.mergeJSONLog` | If the log field content is a JSON string map, append the map fields as part of the log structure | `true` | -| `filter.mergeLogKey` | If set, append the processed log keys under a new root key specified by this variable. | `nil` | -| `filter.useJournal` | If true, the filter reads logs coming in Journald format. | `false` | -| `image.fluent_bit.repository` | Image | `fluent/fluent-bit` | -| `image.fluent_bit.tag` | Image tag | `1.3.7` | -| `image.pullPolicy` | Image pull policy | `Always` | -| `nameOverride` | Override name of app | `nil` | -| `fullnameOverride` | Override full name of app | `nil` | -| `image.pullSecrets` | Specify image pull secrets | `nil` | -| `input.tail.memBufLimit` | Specify Mem_Buf_Limit in tail input | `5MB` | -| `input.tail.parser` | Specify Parser in tail input. | `docker` | -| `input.tail.path` | Specify log file(s) through the use of common wildcards. | `/var/log/containers/*.log` | -| `input.tail.ignore_older` | Ignores files that have been last modified before this time in seconds. Supports m,h,d (minutes, hours,days) syntax. | `` | -| `input.systemd.enabled` | [Enable systemd input](https://docs.fluentbit.io/manual/input/systemd) | `false` | -| `input.systemd.filters.systemdUnit` | Please see https://docs.fluentbit.io/manual/input/systemd | `[docker.service, kubelet.service`, `node-problem-detector.service]` | -| `input.systemd.maxEntries` | Please see https://docs.fluentbit.io/manual/input/systemd | `1000` | -| `input.systemd.readFromTail` | Please see https://docs.fluentbit.io/manual/input/systemd | `true` | -| `input.systemd.stripUnderscores` | Please see https://docs.fluentbit.io/manual/input/systemd | `false` | -| `input.systemd.tag` | Please see https://docs.fluentbit.io/manual/input/systemd | `host.*` | -| `rbac.create` | Specifies whether RBAC resources should be created. | `true` | -| `rbac.pspEnabled` | Specifies whether a PodSecurityPolicy should be created. | `false` | -| `serviceAccount.create` | Specifies whether a ServiceAccount should be created. | `true` | -| `serviceAccount.name` | The name of the ServiceAccount to use. | `NULL` | -| `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | -| `rawConfig` | Raw contents of fluent-bit.conf | `@INCLUDE fluent-bit-service.conf`
`@INCLUDE fluent-bit-input.conf`
`@INCLUDE fluent-bit-filter.conf`
` @INCLUDE fluent-bit-output.conf` | -| `resources` | Pod resource requests & limits | `{}` | -| `securityContext` | [Security settings for a container](https://kubernetes.io/docs/concepts/policy/security-context) | `{}` | -| `podSecurityContext` | [Security settings for a pod](https://kubernetes.io/docs/concepts/policy/security-context) | `{}` | -| `hostNetwork` | Use host's network | `false` | -| `dnsPolicy` | Specifies the dnsPolicy to use | `ClusterFirst` | -| `priorityClassName` | Specifies the priorityClassName to use | `NULL` | -| `tolerations` | Optional daemonset tolerations | `NULL` | -| `nodeSelector` | Node labels for fluent-bit pod assignment | `NULL` | -| `affinity` | Expressions for affinity | `NULL` | -| `metrics.enabled` | Specifies whether a service for metrics should be exposed | `false` | -| `metrics.service.annotations` | Optional metrics service annotations | `NULL` | -| `metrics.service.labels` | Additional labels for the fluent-bit metrics service definition, specified as a map. | None | -| `metrics.service.port` | Port on where metrics should be exposed | `2020` | -| `metrics.service.type` | Service type for metrics | `ClusterIP` | -| `metrics.serviceMonitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | -| `metrics.serviceMonitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | -| `metrics.serviceMonitor.namespace` | Optional namespace in which to create ServiceMonitor | `nil` | -| `metrics.serviceMonitor.interval` | Scrape interval. If not set, the Prometheus default scrape interval is used | `nil` | -| `metrics.serviceMonitor.scrapeTimeout` | Scrape timeout. If not set, the Prometheus default scrape timeout is used | `nil` | -| `trackOffsets` | Specify whether to track the file offsets for tailing docker logs. This allows fluent-bit to pick up where it left after pod restarts but requires access to a `hostPath` | `false` | -| `testFramework.image` | `test-framework` image repository. | `dduportal/bats` | -| `testFramework.tag` | `test-framework` image tag. | `0.4.0` | - - -Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. - -Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, - -```bash -$ helm install --name my-release -f values.yaml stable/fluent-bit -``` - -> **Tip**: You can use the default [values.yaml](values.yaml) - -## Upgrading - -### From < 1.0.0 To >= 1.0.0 - -Values `extraInputs`, `extraFilters` and `extraOutputs` have been removed in version `1.0.0` of the fluent-bit chart. -To add additional entries to the existing sections, please use the `extraEntries.input`, `extraEntries.filter` and `extraEntries.output` values. -For entire sections, please use the `rawConfig` value, inserting blocks of text as desired. - -### From < 1.8.0 to >= 1.8.0 - -Version `1.8.0` introduces the use of release name as full name if it contains the chart name(fluent-bit in this case). E.g. with a release name of `fluent-bit`, this renames the DaemonSet from `fluent-bit-fluent-bit` to `fluent-bit`. The suggested approach is to delete the release and reinstall it. diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/NOTES.txt b/kubernetes/helm_charts/logging/fluent-bit/templates/NOTES.txt deleted file mode 100644 index bbfcc0bef..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/NOTES.txt +++ /dev/null @@ -1,15 +0,0 @@ -fluent-bit is now running. - -{{- if eq .Values.backend.type "forward" }} - -It will forward all container logs to the svc named {{ .Values.backend.forward.host }} on port: {{ .Values.backend.forward.port }} -{{- else if eq .Values.backend.type "es" }} - -It will forward all container logs to the svc named {{ .Values.backend.es.host }} on port: {{ .Values.backend.es.port }} -{{- else if eq .Values.backend.type "http" }} - -It will forward all container logs to the svc named {{ .Values.backend.http.host }} on port: {{ .Values.backend.http.port }} -{{- else if eq .Values.backend.type "splunk" }} - -It will forward all container logs to the svc named {{ .Values.backend.splunk.host }} on port: {{ .Values.backend.splunk.port }} -{{- end }} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl b/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl index e9159a050..189773ee5 100644 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl +++ b/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl @@ -1,10 +1,9 @@ -{{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. */}} {{- define "fluent-bit.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* Create a default fully qualified app name. @@ -12,67 +11,52 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "fluent-bit.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} {{/* Create chart name and version as used by the chart label. */}} {{- define "fluent-bit.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* -Return the appropriate apiVersion for RBAC APIs. +Common labels */}} -{{- define "rbac.apiVersion" -}} -{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}} -rbac.authorization.k8s.io/v1 -{{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}} -rbac.authorization.k8s.io/v1beta1 -{{- else -}} -rbac.authorization.k8s.io/v1alpha1 -{{- end -}} -{{- end -}} +{{- define "fluent-bit.labels" -}} +helm.sh/chart: {{ include "fluent-bit.chart" . }} +{{ include "fluent-bit.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} {{/* -Create the name of the service account to use +Selector labels */}} -{{- define "fluent-bit.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} +{{- define "fluent-bit.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fluent-bit.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} {{/* -Return the appropriate apiGroup for PodSecurityPolicy. -*/}} -{{- define "rbac.pspApiGroup" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions" -}} -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "policy" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for DaemonSet. +Create the name of the service account to use */}} -{{- define "daemonSet.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "apps/v1" -}} -{{- end -}} -{{- end -}} +{{- define "fluent-bit.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-role.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-role.yaml deleted file mode 100644 index 69c79891d..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-role.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.rbac.create -}} -apiVersion: {{ template "rbac.apiVersion" . }} -kind: ClusterRole -metadata: - labels: - app: {{ template "fluent-bit.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - name: {{ template "fluent-bit.fullname" . }} -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get -{{- if .Values.rbac.pspEnabled }} - - apiGroups: - - {{ template "rbac.pspApiGroup" . }} - resources: - - podsecuritypolicies - resourceNames: - - {{ template "fluent-bit.fullname" . }} - verbs: - - use -{{- end }} -{{- end -}} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-rolebinding.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-rolebinding.yaml deleted file mode 100644 index 140e5279b..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/cluster-rolebinding.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.rbac.create -}} -apiVersion: {{ template "rbac.apiVersion" . }} -kind: ClusterRoleBinding -metadata: - labels: - app: {{ template "fluent-bit.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - name: {{ template "fluent-bit.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ template "fluent-bit.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ template "fluent-bit.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end -}} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/config.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/config.yaml deleted file mode 100644 index 37bba568f..000000000 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/config.yaml +++ /dev/null @@ -1,271 +0,0 @@ -{{- if (empty .Values.existingConfigMap) -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "fluent-bit.fullname" . }}-config - labels: - app: {{ template "fluent-bit.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} -data: - fluent-bit-service.conf: | - [SERVICE] - Flush {{ .Values.service.flush }} - Daemon Off - Log_Level {{ .Values.service.logLevel }} - Parsers_File parsers.conf -{{- if .Values.parsers.enabled }} - Parsers_File parsers_custom.conf -{{- end }} -{{- if .Values.metrics.enabled }} - HTTP_Server On - HTTP_Listen 0.0.0.0 - HTTP_Port 2020 -{{- end }} - - fluent-bit-input.conf: | - [INPUT] - Name tail - Path {{ .Values.input.tail.path }} - Parser {{ .Values.input.tail.parser }} - Tag {{ .Values.filter.kubeTag }}.* - Refresh_Interval 5 - Mem_Buf_Limit {{ .Values.input.tail.memBufLimit }} - Skip_Long_Lines On -{{- if .Values.input.tail.ignore_older }} - Ignore_Older {{ .Values.input.tail.ignore_older }} -{{- end }} -{{- if .Values.trackOffsets }} - DB /tail-db/tail-containers-state.db - DB.Sync Normal -{{- end }} -{{- if .Values.input.systemd.enabled }} - [INPUT] - Name systemd - Tag {{ .Values.input.systemd.tag }} -{{- range $value := .Values.input.systemd.filters.systemdUnit }} - Systemd_Filter _SYSTEMD_UNIT={{ $value }} -{{- end }} - Max_Entries {{ .Values.input.systemd.maxEntries }} - Read_From_Tail {{ .Values.input.systemd.readFromTail }} - Strip_Underscores {{ .Values.input.systemd.stripUnderscores }} -{{- end }} -{{ .Values.extraEntries.input | indent 8 }} - -{{- if .Values.audit.enable }} - [INPUT] - Name tail - Path {{ .Values.audit.input.path }} - Parser {{ .Values.audit.input.parser }} - DB /var/log/audit.db - Tag {{ .Values.audit.input.tag }} - Refresh_Interval 5 - Mem_Buf_Limit {{ .Values.audit.input.memBufLimit }} - Buffer_Chunk_Size {{ .Values.audit.input.bufferChunkSize }} - Buffer_Max_Size {{ .Values.audit.input.bufferMaxSize }} - Skip_Long_Lines {{ .Values.audit.input.skipLongLines}} - Key {{ .Values.audit.input.key}} -{{ .Values.extraEntries.audit | indent 8 }} -{{- end }} - - fluent-bit-filter.conf: | - [FILTER] - Name kubernetes - Match {{ .Values.filter.kubeTag }}.* - Kube_Tag_Prefix {{ .Values.filter.kubeTagPrefix }} - Kube_URL {{ .Values.filter.kubeURL }} - Kube_CA_File {{ .Values.filter.kubeCAFile }} - Kube_Token_File {{ .Values.filter.kubeTokenFile }} -{{- if .Values.filter.mergeJSONLog }} - Merge_Log On -{{- end }} - -{{- if .Values.filter.mergeLogKey }} - Merge_Log_Key {{ .Values.filter.mergeLogKey }} -{{- end }} - -{{- if .Values.filter.enableParser }} - K8S-Logging.Parser On -{{- end }} -{{- if .Values.filter.enableExclude }} - K8S-Logging.Exclude On -{{- end }} -{{- if .Values.filter.useJournal }} - Use_Journal On -{{- end }} -{{ .Values.extraEntries.filter | indent 8 }} - - fluent-bit-output.conf: | -{{ if eq .Values.backend.type "test" }} - [OUTPUT] - Name file - Match * - Path /tmp/fluent-bit.log -{{ else if eq .Values.backend.type "forward" }} - [OUTPUT] - Name forward - Match * - Host {{ .Values.backend.forward.host }} - Port {{ .Values.backend.forward.port }} - Retry_Limit False -{{- if .Values.backend.forward.shared_key }} - Shared_Key {{ .Values.backend.forward.shared_key }} -{{- end }} -{{- if .Values.backend.forward.tls }} - tls {{ .Values.backend.forward.tls }} -{{- end }} -{{- if .Values.backend.forward.tls_verify }} - tls.verify {{ .Values.backend.forward.tls_verify }} -{{- end }} -{{- if .Values.backend.forward.tls_debug }} - tls.debug {{ .Values.backend.forward.tls_debug }} -{{- end }} -{{ else if eq .Values.backend.type "es" }} - [OUTPUT] - Name es - Match * - Host {{ .Values.backend.es.host }} - Port {{ .Values.backend.es.port }} - Logstash_Format {{ default "On" .Values.backend.es.logstash_format }} - Retry_Limit {{ .Values.backend.es.retry_limit }} - Type {{ .Values.backend.es.type }} -{{- if .Values.backend.es.time_key }} - Time_Key {{ .Values.backend.es.time_key }} -{{- end }} -{{- if .Values.backend.es.replace_dots }} - Replace_Dots {{ .Values.backend.es.replace_dots }} -{{- end }} -{{- if .Values.backend.es.logstash_prefix }} - Logstash_Prefix {{ .Values.backend.es.logstash_prefix }} -{{- if .Values.backend.es.logstash_prefix_key }} - Logstash_Prefix_Key {{ .Values.backend.es.logstash_prefix_key }} -{{- end }} -{{ else if .Values.backend.es.index }} - Index {{ .Values.backend.es.index }} -{{- end }} -{{- if .Values.backend.es.http_user }} - HTTP_User {{ .Values.backend.es.http_user }} -{{- if .Values.backend.es.http_passwd }} - HTTP_Passwd {{ .Values.backend.es.http_passwd }} -{{- else }} - HTTP_Passwd ${HTTP_PASSWORD} -{{- end }} -{{- end }} -{{if eq .Values.backend.es.tls "on" }} - tls {{ .Values.backend.es.tls }} - tls.verify {{ .Values.backend.es.tls_verify }} - tls.debug {{ .Values.backend.es.tls_debug }} -{{- if .Values.backend.es.tls_ca }} - tls.ca_file /secure/es-tls-ca.crt -{{- end }} -{{- end }} -{{ else if eq .Values.backend.type "splunk" }} - [OUTPUT] - Name splunk - Match * - Host {{ .Values.backend.splunk.host }} - Port {{ .Values.backend.splunk.port }} - Splunk_Token {{ .Values.backend.splunk.token }} - Splunk_Send_Raw {{ .Values.backend.splunk.send_raw}} - TLS {{ .Values.backend.splunk.tls }} - TLS.Verify {{ .Values.backend.splunk.tls_verify }} - tls.debug {{ .Values.backend.splunk.tls_debug }} - Message_Key {{ .Values.backend.splunk.message_key }} -{{ else if eq .Values.backend.type "stackdriver" }} - [OUTPUT] - Name stackdriver - Match * - resource global -{{- if .Values.backend.stackdriver.google_service_credentials }} - google_service_credentials /secure/google_service_credentials.json -{{- else }} - service_account_email {{ .Values.backend.stackdriver.service_account_email }} - service_account_secret {{ .Values.backend.stackdriver.service_account_secret }} -{{- end }} -{{ else if eq .Values.backend.type "http" }} - [OUTPUT] - Name http - Match * - Host {{ .Values.backend.http.host }} - Port {{ .Values.backend.http.port }} - URI {{ .Values.backend.http.uri }} -{{- if .Values.backend.http.http_user }} - HTTP_User {{ .Values.backend.http.http_user }} - HTTP_Passwd {{ .Values.backend.http.http_passwd }} -{{- end }} - tls {{ .Values.backend.http.tls }} - tls.verify {{ .Values.backend.http.tls_verify }} - tls.debug {{ .Values.backend.http.tls_debug }} -{{- if .Values.backend.http.proxy }} - Proxy {{ .Values.backend.http.proxy }} -{{- end }} - Format {{ .Values.backend.http.format }} -{{- end }} -{{- if .Values.backend.http.json_date_format }} - json_date_format {{ .Values.backend.http.json_date_format }} -{{- end }} -{{- range .Values.backend.http.headers }} - Header {{ . }} -{{- end }} -{{ .Values.extraEntries.output | indent 8 }} - - - fluent-bit.conf: | -{{ .Values.rawConfig | indent 4 }} - - parsers.conf: | -{{- if .Values.parsers.regex }} -{{- range .Values.parsers.regex }} - [PARSER] - Name {{ .name }} - Format regex - Regex {{ .regex }} -{{- if .timeKey }} - Time_Key {{ .timeKey }} -{{- end }} -{{- if .timeFormat }} - Time_Format {{ .timeFormat }} -{{- end }} -{{ end }} -{{- end }} -{{- if .Values.parsers.json }} -{{- range .Values.parsers.json }} - [PARSER] - Name {{ .name }} - Format json -{{- if .timeKeep }} - Time_Keep {{ .timeKeep }} -{{- end }} -{{- if .timeKey }} - Time_Key {{ .timeKey }} -{{- end }} -{{- if .timeFormat }} - Time_Format {{ .timeFormat }} -{{- end }} -{{- if .decodeFieldAs }} - Decode_Field_As {{ .decodeFieldAs }} {{ .decodeField | default "log" }} -{{- end}} -{{- if .extraEntries }} -{{ .extraEntries | indent 8 }} -{{- end }} -{{ end }} -{{- end }} -{{- if .Values.parsers.logfmt }} -{{- range .Values.parsers.logfmt }} - [PARSER] - Name {{ .name }} - Format logfmt -{{- if .timeKey }} - Time_Key {{ .timeKey }} -{{- end }} -{{- if .timeFormat }} - Time_Format {{ .timeFormat }} -{{- end }} -{{- if .extraEntries }} -{{ .extraEntries | indent 8 }} -{{- end }} -{{ end }} -{{- end }} - -{{- end -}} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml new file mode 100644 index 000000000..010761e6e --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: v1 +data: + filter-kubernetes.conf: | + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Kube_Tag_Prefix kube.var.log.containers. + # Merge application log to kubernetes structure + Merge_Log On + # Disabling merge key as all the fileds will be in the root + # Else all merged content will be under + # log_processed: + # { + # } + # Merge_Log_Key log_processed + # + # Adding custom parsor not to do any json parse + Merge_Parser catchall + # Don't need merged logs + # More over, we're using custom parser, which catch all and tag it as log + Keep_Log Off + # Annotate pod to select the parser + # annotaions: + # fluentbit.io/parsers: nginx2 + K8S-Logging.Parser On + # Disable fluent-bit from parsing logs + # annotaions: + # fluentbit.io/exclude: true + K8S-Logging.Exclude On + [FILTER] + # Discard all health debug and info logs + Name grep + Match kube.* + exclude message /^.*(?:debug|info|GET (\/service)?\/health).*$ + + fluent-bit.conf: | + [SERVICE] + Flush 1 + Log_Level WARNING + Daemon off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + storage.path /mnt/log/flb-storage/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 120M + # How many chunks from file system, which is in que to flush to es, has to be in memory(warm) + storage.max_chunks_up 50 + + @INCLUDE input-kubernetes.conf + @INCLUDE filter-kubernetes.conf + @INCLUDE output-elasticsearch.conf + input-kubernetes.conf: | + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser docker + DB /mnt/log/flb_kube.db + Mem_Buf_Limit 120MB + # Set the initial buffer size to read files data. + # This value is used too to increase buffer size. + Buffer_Chunk_Size 256k + # Set the limit of the buffer size per monitored file. + # When a buffer needs to be increased (e.g: very long lines), + # this value is used to restrict how much the memory buffer can grow. + # If reading a file exceed this limit, the file is removed from the monitored file list + Buffer_Max_Size 512k + Skip_Long_Lines On + Refresh_Interval 1 + storage.type filesystem + # Ignore older logs of 30m + Ignore_Older 30m + output-elasticsearch.conf: | + [OUTPUT] + Name es + Match * + Host ${FLUENT_ELASTICSEARCH_HOST} + Port ${FLUENT_ELASTICSEARCH_PORT} + Logstash_Format On + Replace_Dots On + Retry_Limit 10 + Type _doc + # Networking Setup + net.connect_timeout 10 + # net.source_address 127.0.0.1 + net.keepalive on + net.keepalive_idle_timeout 100 + net.keepalive_max_recycle 200 + # + # Limit the maximum number of Chunks in the filesystem for the current output logical destination. + # After this LRU will kick in; that means LeastRecentlyUsed will get deleted + # 1 chunk =~ 2MB + # This config has some issues; see https://github.com/fluent/fluent-bit/issues/2688 + storage.total_limit_size 20G + parsers.conf: | + [PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?