From 3a907645d7e7b05578a2776f42d5dca838f801db Mon Sep 17 00:00:00 2001 From: Steven Smith Date: Thu, 29 Jan 2026 13:46:51 -0500 Subject: [PATCH] Resolves ADV errors in Troubleshooting guide --- ...authentication-troubleshooting-issues.adoc | 7 +- modules/authentication-troubleshooting.adoc | 3 +- modules/check-backend-storage-running.adoc | 22 ++ modules/check-data-replication.adoc | 26 ++ modules/check-resource-allocation.adoc | 57 ++++ modules/clair-dependencies-update.adoc | 38 +++ modules/clair-troubleshooting-issues.adoc | 153 +---------- modules/connectivity-networking.adoc | 24 ++ modules/creating-registry-infra-node.adoc | 74 +---- modules/database-troubleshooting-issues.adoc | 259 +----------------- modules/database-troubleshooting.adoc | 3 +- modules/enabling-debug-mode-clair.adoc | 56 ++++ modules/geo-repl-troubleshooting-issues.adoc | 82 +----- modules/getting-support.adoc | 62 +---- modules/health-check-quay.adoc | 28 +- modules/instance-endpoint-quay.adoc | 19 ++ modules/interact-with-database.adoc | 55 ++++ modules/mirroring-intro.adoc | 3 +- ...aining-configuration-information-quay.adoc | 109 ++++++++ modules/obtaining-db-config-info.adoc | 27 ++ modules/obtaining-log-information-quay.adoc | 30 ++ .../obtaining-quay-config-information.adoc | 145 +--------- modules/obtaining-quay-logs.adoc | 94 +------ .../obtaining-verbose-container-pod-logs.adoc | 55 ++++ ...repo-mirroring-troubleshooting-issues.adoc | 110 +------- ...etting-superuser-password-on-operator.adoc | 7 +- modules/running-ldap-debug-mode.adoc | 3 +- modules/running-operator-debug-mode.adoc | 3 +- modules/running-quay-debug-mode-intro.adoc | 8 +- modules/running-quay-debug-mode.adoc | 5 +- modules/storage-troubleshooting-issues.adoc | 3 +- modules/storage-troubleshooting.adoc | 3 + modules/support-knowledgebase-search.adoc | 26 ++ modules/support-submitting-a-case.adoc | 31 +++ modules/troubleshooting-components.adoc | 7 +- ...oubleshooting-crashloop-backoff-state.adoc | 92 +++++++ .../troubleshooting-forgotten-passwords.adoc | 15 +- troubleshooting_quay/master.adoc | 100 +++---- 38 files changed, 769 insertions(+), 1075 deletions(-) create mode 100644 modules/check-backend-storage-running.adoc create mode 100644 modules/check-data-replication.adoc create mode 100644 modules/check-resource-allocation.adoc create mode 100644 modules/clair-dependencies-update.adoc create mode 100644 modules/connectivity-networking.adoc create mode 100644 modules/enabling-debug-mode-clair.adoc create mode 100644 modules/instance-endpoint-quay.adoc create mode 100644 modules/interact-with-database.adoc create mode 100644 modules/obtaining-configuration-information-quay.adoc create mode 100644 modules/obtaining-db-config-info.adoc create mode 100644 modules/obtaining-log-information-quay.adoc create mode 100644 modules/obtaining-verbose-container-pod-logs.adoc create mode 100644 modules/support-knowledgebase-search.adoc create mode 100644 modules/support-submitting-a-case.adoc create mode 100644 modules/troubleshooting-crashloop-backoff-state.adoc diff --git a/modules/authentication-troubleshooting-issues.adoc b/modules/authentication-troubleshooting-issues.adoc index 8e3089843..db8f74401 100644 --- a/modules/authentication-troubleshooting-issues.adoc +++ b/modules/authentication-troubleshooting-issues.adoc @@ -1,8 +1,9 @@ -:_mod-docs-content-type: CONCEPT +:_mod-docs-content-type: PROCEDURE [id="authentication-troubleshooting-issues"] = Troubleshooting {productname} authentication and authorization issues for specific users -Use the following procedure to troubleshoot authentication and authorization issues for specific users. +[role="_abstract"] +To troubleshoot {productname} authentication and authorization issues for specific users, you can exec into the {productname} pod or container and query the `federatedlogin` and `user` tables to verify user records. .Procedure @@ -16,7 +17,6 @@ quay=# select * from federatedlogin; ---- + .Example output -+ [source,terminal] ---- id | user_id | service_id | service_ident | metadata_json @@ -38,7 +38,6 @@ quay=# select username, email from "user"; ---- + .Example output -+ [source,terminal] ---- username | email diff --git a/modules/authentication-troubleshooting.adoc b/modules/authentication-troubleshooting.adoc index debd6a5a0..a50e9800b 100644 --- a/modules/authentication-troubleshooting.adoc +++ b/modules/authentication-troubleshooting.adoc @@ -2,7 +2,8 @@ [id="authentication-troubleshooting"] = Troubleshooting {productname} authentication -Authentication and authorization is crucial for secure access to {productname}. Together, they safeguard sensitive container images, verify user identities, enforce access controls, facilitate auditing and accountability, and enable seamless integration with external identity providers. By prioritizing authentication, organizations can bolster the overall security and integrity of their container registry environment. +[role="_abstract"] +Authentication and authorization secure access to {productname} and safeguard container images, verify identities, and enforce access controls. The following authentication methods are supported by {productname}: diff --git a/modules/check-backend-storage-running.adoc b/modules/check-backend-storage-running.adoc new file mode 100644 index 000000000..9bbe68f92 --- /dev/null +++ b/modules/check-backend-storage-running.adoc @@ -0,0 +1,22 @@ +:_mod-docs-content-type: CONCEPT +[id="check-backend-storage-running"] += Checking the status of your backend storage + +[role="_abstract"] +To check the status of your {product-title} backend storage and verify access, you can use provider dashboards and CLIs for AWS, GCS, NooBaa, ODF, Ceph, Azure, or OpenStack Swift. Ensure all {product-title} instances have access to all S3 storage backends. + +* *Amazon Web Service Storage (AWS)*. Check the AWS S3 service health status on the link:https://health.aws.amazon.com/health/status[AWS Service Health Dashboard]. Validate your access to S3 by listing objects in a known bucket using the `aws` CLI or SDKs. + +* *Google Cloud Storage (GCS)*. Check the link:https://status.cloud.google.com/[Google Cloud Status Dashboard] for the status of the GCS service. Verify your access to GCS by listing objects in a known bucket using the Google Cloud SDK or GCS client libraries. + +* *NooBaa*. Check the NooBaa management console or administrative interface for any health or status indicators. Ensure that the NooBaa services and related components are running and accessible. Verify access to NooBaa by listing objects in a known bucket using the NooBaa CLI or SDK. + +* **{odf}**. Check the {ocp} Console or management interface for the status of the {odf} components. Verify the availability of {odf} S3 interface and services. Ensure that the {odf} services are running and accessible. Validate access to {odf} S3 by listing objects in a known bucket using the appropriate S3-compatible SDK or CLI. + +* **Ceph**. Check the status of Ceph services, including Ceph monitors, OSDs, and RGWs. Validate that the Ceph cluster is healthy and operational. Verify access to Ceph object storage by listing objects in a known bucket using the appropriate Ceph object storage API or CLI. + +* **Azure Blob Storage**. Check the link:https://azure.status.microsoft/en-us/status[Azure Status Dashboard] to see the health status of the Azure Blob Storage service. Validate your access to Azure Blob Storage by listing containers or objects using the Azure CLI or Azure SDKs. + +* **OpenStack Swift**. Check the link:https://www.ibm.com/docs/ro/cmwo/4.3.0.0?topic=services-checking-status[OpenStack Status] page to verify the status of the OpenStack Swift service. Ensure that the Swift services, like the proxy server, container servers, object servers, are running and accessible. Validate your access to Swift by listing containers or objects using the appropriate Swift CLI or SDK. + +After checking the status of your backend storage, ensure that all {productname} instances have access to all s3 storage backends. \ No newline at end of file diff --git a/modules/check-data-replication.adoc b/modules/check-data-replication.adoc new file mode 100644 index 000000000..ae81147f4 --- /dev/null +++ b/modules/check-data-replication.adoc @@ -0,0 +1,26 @@ +:_mod-docs-content-type: PROCEDURE +[id="check-data-replication"] += Checking data replication in backend buckets + +[role="_abstract"] +To ensure that your {product-title} data is properly replicated in all backend buckets, you can use the `aws CLI` to list objects in the bucket. Run `aws s3 ls` with `--recursive`, `--human-readable`, and `--summarize` to verify replication. + +.Prerequisites + +* You have installed the `aws` CLI. + +.Procedure + +. Enter the following command to ensure that your data is replicated in all backend buckets: ++ +[source,terminal] +---- +$ aws --profile quay_prod_s3 --endpoint=http://10.0.x.x:port s3 ls ocp-quay --recursive --human-readable --summarize +---- ++ +.Example output +[source,terminal] +---- +Total Objects: 17996 +Total Size: 514.4 GiB +---- \ No newline at end of file diff --git a/modules/check-resource-allocation.adoc b/modules/check-resource-allocation.adoc new file mode 100644 index 000000000..077489b20 --- /dev/null +++ b/modules/check-resource-allocation.adoc @@ -0,0 +1,57 @@ +:_mod-docs-content-type: PROCEDURE +[id="check-resource-allocation"] += Checking resource allocation + +[role="_abstract"] +To check resource allocation for your {productname} deployment and monitor disk, CPU, and memory usage, you can use `oc exec` or `podman exec` for disk usage and `oc adm top pods` or `podman stats` for other resources. + +.Procedure + +. Obtain a list of running containers. + +. Monitor disk usage of your {productname} deployment. + +.. If you are using the {productname} Operator on {ocp}, enter the following command: ++ +[source,terminal] +---- +$ oc exec -it -- df -ah +---- + +.. If you are using a standalone deployment of {productname}, enter the following command: ++ +[source,terminal] +---- +$ podman exec -it df -ah +---- + +. Monitor other resource usage. + +.. Enter the following command to check resource allocation on a {productname} Operator deployment: ++ +[source,terminal] +---- +$ oc adm top pods +---- + +.. Enter the following command to check the status of a specific pod on a standalone deployment of {productname}: ++ +[source,terminal] +---- +$ podman pod stats +---- + +.. Enter the following command to check the status of a specific container on a standalone deployment of {productname}: ++ +[source,terminal] +---- +$ podman stats +---- ++ +The following information is returned: ++ +* *CPU %*. The percentage of CPU usage by the container since the last measurement. This value represents the container's share of the available CPU resources. +* *MEM USAGE / LIMIT*. The current memory usage of the container followed by its memory limit. The values are displayed in the format `current_usage / memory_limit`. For example, `300.4MiB / 7.795GiB` indicates that the container is currently using 300.4 megabytes of memory out of a limit of 7.795 gigabytes. +* *MEM %*. The percentage of memory usage by the container in relation to its memory limit. +* *NET I/O*. The network I/O (input/output) statistics of the container. It displays the amount of data transmitted and received by the container over the network. The values are displayed in the format: `transmitted_bytes / received_bytes`. +* *BLOCK I/O*. The block I/O (input/output) statistics of the container. It represents the amount of data read from and written to the block devices (for example, disks) used by the container. The values are displayed in the format `read_bytes / written_bytes`. \ No newline at end of file diff --git a/modules/clair-dependencies-update.adoc b/modules/clair-dependencies-update.adoc new file mode 100644 index 000000000..0418ebed3 --- /dev/null +++ b/modules/clair-dependencies-update.adoc @@ -0,0 +1,38 @@ +:_mod-docs-content-type: PROCEDURE +[id="clair-dependencies-update"] +== Updating Clair scanner and its dependencies + +[role="_abstract"] +To ensure your Clair scanner is up to date and avoid issues with newer image formats in {product-title}, you can check your Clair version using oc logs for OpenShift or podman logs for standalone deployments. Use the latest Clair version for support for newer formats and vulnerability database updates. + +[NOTE] +==== +Checking Clair logs can also be used to check if there are any errors from the updaters microservice in your Clair logs. By default, Clair updates the vulnerability database every 30 minutes. +==== + +.Procedure + +. Check your version of Clair. + +.. If you are running Clair on {productname-ocp}, enter the following command: ++ +[source,terminal] +---- +$ oc logs clair-pod +---- + +.. If you are running a standalone deployment of {productname} and using a Clair container, enter the following command: ++ +[source,terminal] +---- +$ podman logs clair-container +---- ++ +.Example output ++ +[source,terminal] +---- +"level":"info", +"component":"main", +"version":"v4.5.1", +---- \ No newline at end of file diff --git a/modules/clair-troubleshooting-issues.adoc b/modules/clair-troubleshooting-issues.adoc index 0561730af..7f9819c24 100644 --- a/modules/clair-troubleshooting-issues.adoc +++ b/modules/clair-troubleshooting-issues.adoc @@ -1,8 +1,9 @@ -:_mod-docs-content-type: PROCEDURE +:_mod-docs-content-type: CONCEPT [id="clair-troubleshooting-issues"] = Troubleshooting Clair issue -Use the following procedures to troubleshoot Clair. +[role="_abstract"] +To troubleshoot Clair vulnerability scanning in {product-title} and resolve scan issues, you can verify image compatibility, allowlist Clair updaters when using a proxy, check the Clair config.yaml, and inspect image metadata. [id="verify-image-compatibility"] == Verifying image compatibility @@ -16,101 +17,6 @@ For more information, see link:https://access.redhat.com/documentation/en-us/red If you are using Clair behind a proxy configuration, you must allowlist the updaters in your proxy or firewall configuration. For more information about updater URLs, see link:https://access.redhat.com/documentation/en-us/red_hat_quay/{producty}/html/vulnerability_reporting_with_clair_on_red_hat_quay/clair-concepts#clair-updater-urls[Clair updater URLs]. -[id="clair-dependencies-update"] -== Updating Clair scanner and its dependencies - -Ensure that you are using the latest version of Clair security scanner. Outdated versions might lack support for newer image formats, or might have known issues. - -Use the following procedure to check your version of Clair. - -[NOTE] -==== -Checking Clair logs can also be used to check if there are any errors from the updaters microservice in your Clair logs. By default, Clair updates the vulnerability database every 30 minutes. -==== - -.Procedure - -. Check your version of Clair. - -.. If you are running Clair on {productname-ocp}, enter the following command: -+ -[source,terminal] ----- -$ oc logs clair-pod ----- - -.. If you are running a standalone deployment of {productname} and using a Clair container, enter the following command: -+ -[source,terminal] ----- -$ podman logs clair-container ----- -+ -.Example output -+ -[source,terminal] ----- -"level":"info", -"component":"main", -"version":"v4.5.1", ----- - -[id="enabling-debug-mode-clair"] -== Enabling debug mode for Clair - -By default, debug mode for Clair is disabled. You can enable debug mode for Clair by updating your `clair-config.yaml` file. - -.Prerequisites - -* For Clair on {productname-ocp} deployments, you must link:https://docs.redhat.com/en/documentation/red_hat_quay/3.12/html-single/vulnerability_reporting_with_clair_on_red_hat_quay/index#custom-clair-configuration-managed-database[Running a custom Clair configuration with a managed Clair database]. - -Use the following procedure to enable debug mode for Clair. - -.Procedure - -. Update your `clair-config.yaml` file to include the debug option. - -.. On standalone {productname} deployments: - -... Add the following configuration field to your `clair-config.yaml` file: -+ -[source,yaml] ----- -log_level: debug ----- - -... Restart your Clair deployment by entering the following command: -+ -[source,terminal] ----- -$ podman restart ----- - -.. On {productname-ocp} deployments: - -... On the {ocp} web console, click *Operators* -> *Installed Operators* -> *Quay Registry*. - -... Click the name of your registry, for example, *Example Registry*. You are redirected to the *Details* page of your registry. - -... Click the Config Bundle Secret, for example, *example-registry-config-bundle-xncls*. - -... Confirm that you are running a custom Clair configuration by looking for the `clair-config.yaml` file under the *Data* section of the *Details* page of your secret. - -... If you have a `clair-config.yaml` file, click *Actions* -> *Edit Secret*. If you do not, see "Running a custom Clair configuration with a managed Clair database". - -... Update your `clair-config.yaml` file to include the `log_level: debug` configuration variable. For example: -+ -[source,yaml] ----- -log_level: debug ----- - -... Click *Save*. - -... You can check the status of your Clair deployment by clicking *Workloads* -> *Pods*. The `clair-app` pod should report `1/1` under the *Ready* category. - -... You can confirm that Clair is returning debugging information by clicking the *clair-app* pod that is ready -> *Logs*. - [id="checking-clair-configuration"] == Checking Clair configuration @@ -119,55 +25,4 @@ Check your Clair `config.yaml` file to ensure that there are no misconfiguration [id="inspect-image-metadata"] == Inspect image metadata -In some cases, you might receive an *Unsupported* message. This might indicate that the scanner is unable to extract the necessary metadata from the image. Check if the image metadata is properly formatted and accessible. - -[role="_additional-resources"] -.Additional resources - -For more information, see link:https://access.redhat.com/articles/7018077[Troubleshooting Clair]. - - -//// -[id="check-logs-updaters-errors"] -== Checking logs for updaters errors - -Check if there are any errors from the updaters microservice in your Clair logs. By default, Clair updates the vulnerability database every 30 minutes. - -Use the following procedure to check your Clair logs. - -.Procedure - -. Check your Clair logs. - -.. If you are running Clair on the {productname} Operator, enter the following command: -+ -[source,terminal] ----- -$ oc logs clair-pod ----- - -.. If you are running a standalone deployment of {productname} and using a Clair container, enter the following command: -+ -[source,terminal] ----- -$ podman logs clair-container ----- - - -[id="updating-cve-database"] -== Updating the CVE database - -Updating the CVE database can be a memory and CPU intensive task, especially if there are several CVEs that must be parsed. If the resources are exhausted during this process, the system kernel can stop the offending process. This should be visible in Docker logs, Podman logs, or in the system journal. For example: - -[source,terminal] ----- -May 14 21:48:14 vm-mtr3-live-k8s-00-ranchernode-4 kernel: [36611.338195] [26556] 0 26556 734467 386889 4165632 0 937 clair - -May 14 21:48:14 vm-mtr3-live-k8s-00-ranchernode-4 kernel: [36611.338227] Memory cgroup out of memory: Kill process 26556 (clair) score 1922 or sacrifice child - -May 14 21:48:14 vm-mtr3-live-k8s-00-ranchernode-4 kernel: [36611.339573] Killed process 26556 (clair) total-vm:2937868kB, anon-rss:1536364kB, file-rss:11192kB, shmem-rss:0kB - -May 14 21:48:14 vm-mtr3-live-k8s-00-ranchernode-4 kernel: [36611.396171] oom_reaper: reaped process 26556 (clair), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB ----- -//// - +In some cases, you might receive an *Unsupported* message. This might indicate that the scanner is unable to extract the necessary metadata from the image. Check if the image metadata is properly formatted and accessible. \ No newline at end of file diff --git a/modules/connectivity-networking.adoc b/modules/connectivity-networking.adoc new file mode 100644 index 000000000..4fa961563 --- /dev/null +++ b/modules/connectivity-networking.adoc @@ -0,0 +1,24 @@ +:_mod-docs-content-type: PROCEDURE +[id="connectivity-networking"] += Checking the connectivity between {productname} and the database pod + +[role="_abstract"] +To check the connectivity between your {productname} instance and the database pod and troubleshoot connection issues, you can use `oc exec` for Operator deployments or `podman exec` for standalone deployments. + +.Procedure + +. Check the connectivity between {productname} and the database pod. + +.. If you are using the {productname} Operator on {ocp}, enter the following command: ++ +[source,terminal] +---- +$ oc exec -it _quay_pod_name_ -- curl -v telnet://:5432 +---- + +.. If you are using a standalone deployment of {productname}, enter the following command: ++ +[source,terminal] +---- +$ podman exec -it curl -v telnet://:5432 +---- \ No newline at end of file diff --git a/modules/creating-registry-infra-node.adoc b/modules/creating-registry-infra-node.adoc index f7f25abe6..338945ee3 100644 --- a/modules/creating-registry-infra-node.adoc +++ b/modules/creating-registry-infra-node.adoc @@ -25,78 +25,6 @@ To create a {productname} registry that runs on infrastructure nodes, you can cr . Click *Create*. -//// -. The following condition is reported: `Condition: RolloutBlocked`. This occurs because all pods for the registry must include the `node-role.kubernetes.io/infra` nodeSelector and toleration. Apply the `node-role.kubernetes.io/infra` nodeSelector and toleration to all pods by entering the following command: -+ -[source,terminal] ----- -$ for deploy in $(oc get deployments -n -o name | grep -E 'example-registry-(clair|quay)'); do - oc patch $deploy -n annotated_namespace --type='strategic' -p '{ - "spec": { - "template": { - "spec": { - "nodeSelector": { - "node-role.kubernetes.io/infra": "" - }, - "tolerations": [ - { - "key": "node-role.kubernetes.io/infra", - "operator": "Exists", - "effect": "NoSchedule" - } - ] - } - } - } - }' -done ----- -+ -.Example output -+ -[source,terminal] ----- -deployment.apps/example-registry-clair-app patched -deployment.apps/example-registry-clair-postgres patched -deployment.apps/example-registry-quay-app patched -deployment.apps/example-registry-quay-database patched -deployment.apps/example-registry-quay-mirror patched -deployment.apps/example-registry-quay-redis patched ----- - -. Ensure that all pods include the `node-role.kubernetes.io/infra` nodeSelector and toleration by entering the following command: -+ -[source,terminal] ----- -$ for deploy in $(oc get deployments -n -o name | grep example-registry); do - echo $deploy - oc get -n $deploy -o yaml | grep -A5 nodeSelector - oc get -n $deploy -o yaml | grep -A5 tolerations -done ----- -+ -.Example output -+ -[source,terminal] ----- -... -example-registry-clair-app - nodeSelector: - node-role.kubernetes.io/infra: "" - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - serviceAccount: example-registry-clair-app - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/infra - operator: Exists - volumes: - - configMap: -... ----- -//// - . Optional: Confirm that the pods are running on infra nodes. .. List all `Quay`-related pods along with the nodes that they are scheduled on by entering the following command: @@ -105,7 +33,6 @@ example-registry-clair-app ---- $ oc get pods -n -o wide | grep example-registry ---- -+ [source,terminal] ---- ... @@ -121,6 +48,7 @@ example-registry-clair-app-5f95d685bd-dgjf6 1/1 Running 0 $ oc get nodes -l node-role.kubernetes.io/infra -o name ---- + +.Example output ---- node/example-cluster-new-c5qqp-worker-b-4zxx5.c.quay-devel.internal modified node/example-cluster-new-c5qqp-worker-b-kz6jn.c.quay-devel.internal modified diff --git a/modules/database-troubleshooting-issues.adoc b/modules/database-troubleshooting-issues.adoc index 0a0a4aa75..9409c394c 100644 --- a/modules/database-troubleshooting-issues.adoc +++ b/modules/database-troubleshooting-issues.adoc @@ -2,260 +2,5 @@ [id="database-troubleshooting-issues"] = Troubleshooting {productname} database issues -Use the following procedures to troubleshoot the PostgreSQL database. - -//// -[id="checking-deployment-type"] -== Checking the type of deployment - -Check whether your database is deployed as a container on a virtual machine, or deployed on {ocp} as a pod. - -[id="checking-container-pod-status"] -== Checking the container or pod status - -Use the following procedure to check the status of the database pod or container. - -.Procedure - -. Enter the following command to check the status of the pod or container. - -.. If you are using the {productname} Operator on {ocp}, enter the following command: -+ -[source,terminal] ----- -$ oc get pods ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman ps -a ----- -//// - -[id="interact-with-database"] -== Interacting with the {productname} database - -Use the following procedure to interact with the PostgreSQL database. - -[WARNING] -==== -Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. -==== - -[NOTE] -==== -Interacting with the PostgreSQL database can also be used to troubleshoot authorization and authentication issues. -==== - -.Procedure - -. Exec into the {productname} database. - -.. Enter the following commands to exec into the {productname} database pod on {ocp}: -+ -[source,terminal] ----- -$ oc exec -it -- psql ----- - -.. Enter the following command to exec into the {productname} database on a standalone deployment: -+ -[source,terminal] ----- -$ sudo podman exec -it /bin/bash ----- - -. Enter the PostgreSQL shell. -+ -[WARNING] -==== -Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. -==== - -.. If you are using the {productname} Operator, enter the following command to enter the PostgreSQL shell: -+ -[source,terminal] ----- -$ oc rsh psql -U your_username -d your_database_name ----- - -.. If you are on a standalone {productname} deployment, enter the following command to enter the PostgreSQL shell: -+ -[source,terminal] ----- -bash-4.4$ psql -U your_username -d your_database_name ----- - -[id="troubleshooting-crashloop-backoff-state"] -== Troubleshooting crashloopbackoff states - -Use the following procedure to troueblshoot `crashloopbackoff` states. - -.Procedure - -. If your container or pod is in a `crashloopbackoff` state, you can enter the following commands. - -.. Enter the following command to scale down the {productname} Operator: -+ -[source,terminal] ----- -$ oc scale deployment/quay-operator.v3.8.z --replicas=0 ----- -+ -.Example output -+ -[source,terminal] ----- -deployment.apps/quay-operator.v3.8.z scaled ----- - -.. Enter the following command to scale down the {productname} database: -+ -[source,terminal] ----- -$ oc scale deployment/ --replicas=0 ----- -+ -.Example output -+ -[source,terminal] ----- -deployment.apps/ scaled ----- - -.. Enter the following command to edit the {productname} database: -+ -[WARNING] -==== -Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. -==== -+ -[source,terminal] ----- -$ oc edit deployment ----- -+ -[source,yaml] ----- -... - template: - metadata: - creationTimestamp: null - labels: - quay-component: - quay-operator/quayregistry: quay-operator.v3.8.z - spec: - containers: - - env: - - name: POSTGRESQL_USER - value: postgres - - name: POSTGRESQL_DATABASE - value: postgres - - name: POSTGRESQL_PASSWORD - value: postgres - - name: POSTGRESQL_ADMIN_PASSWORD - value: postgres - - name: POSTGRESQL_MAX_CONNECTIONS - value: "1000" - image: registry.redhat.io/rhel8/postgresql-10@sha256:a52ad402458ec8ef3f275972c6ebed05ad64398f884404b9bb8e3010c5c95291 - imagePullPolicy: IfNotPresent - name: postgres - command: ["/bin/bash", "-c", "sleep 86400"] <1> -... ----- -<1> Add this line in the same indentation. -+ -.Example output -+ -[source,terminal] ----- -deployment.apps/ edited ----- - -.. Execute the following command inside of your ``: -+ -[source,terminal] ----- -$ oc exec -it -- cat /var/lib/pgsql/data/userdata/postgresql/logs/* /path/to/desired_directory_on_host ----- - -[id="connectivity-networking"] -== Checking the connectivity between {productname} and the database pod - -Use the following procedure to check the connectivity between {productname} and the database pod - -.Procedure - -. Check the connectivity between {productname} and the database pod. - -.. If you are using the {productname} Operator on {ocp}, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it _quay_pod_name_ -- curl -v telnet://:5432 ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman exec -it curl -v telnet://:5432 ----- - -[id="check-resource-allocation"] -== Checking resource allocation - -Use the following procedure to check resource allocation. - -.Procedure - -. Obtain a list of running containers. - -. Monitor disk usage of your {productname} deployment. - -.. If you are using the {productname} Operator on {ocp}, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it -- df -ah ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman exec -it df -ah ----- - -. Monitor other resource usage. - -.. Enter the following command to check resource allocation on a {productname} Operator deployment: -+ -[source,terminal] ----- -$ oc adm top pods ----- - -.. Enter the following command to check the status of a specific pod on a standalone deployment of {productname}: -+ -[source,terminal] ----- -$ podman pod stats ----- - -.. Enter the following command to check the status of a specific container on a standalone deployment of {productname}: -+ -[source,terminal] ----- -$ podman stats ----- -+ -The following information is returned: -+ -* *CPU %*. The percentage of CPU usage by the container since the last measurement. This value represents the container's share of the available CPU resources. -* *MEM USAGE / LIMIT*. The current memory usage of the container followed by its memory limit. The values are displayed in the format `current_usage / memory_limit`. For example, `300.4MiB / 7.795GiB` indicates that the container is currently using 300.4 megabytes of memory out of a limit of 7.795 gigabytes. -* *MEM %*. The percentage of memory usage by the container in relation to its memory limit. -* *NET I/O*. The network I/O (input/output) statistics of the container. It displays the amount of data transmitted and received by the container over the network. The values are displayed in the format: `transmitted_bytes / received_bytes`. -* *BLOCK I/O*. The block I/O (input/output) statistics of the container. It represents the amount of data read from and written to the block devices (for example, disks) used by the container. The values are displayed in the format `read_bytes / written_bytes`. \ No newline at end of file +[role="_abstract"] +To troubleshoot {productname} database issues and resolve common errors like `database connection refused` or `authentication failed`, you can check the database logs, verify credentials, and ensure the database is running. \ No newline at end of file diff --git a/modules/database-troubleshooting.adoc b/modules/database-troubleshooting.adoc index dc426c834..a2e8caeea 100644 --- a/modules/database-troubleshooting.adoc +++ b/modules/database-troubleshooting.adoc @@ -2,7 +2,8 @@ [id="database-troubleshooting"] = Troubleshooting the {productname} database -The PostgreSQL database used for {productname} store various types of information related to container images and their management. Some of the key pieces of information that the PostgreSQL database stores includes: +[role="_abstract"] +To troubleshoot the {product-title} database and resolve connectivity, configuration, or resource issues, you can check deployment type, pod or container status, logs, connectivity, and configuration. You can also examine resource allocation and interact with the PostgreSQL database. * *Image Metadata*. The database stores metadata associated with container images, such as image names, versions, creation timestamps, and the user or organization that owns the image. This information allows for easy identification and organization of container images within the registry. diff --git a/modules/enabling-debug-mode-clair.adoc b/modules/enabling-debug-mode-clair.adoc new file mode 100644 index 000000000..ce705822d --- /dev/null +++ b/modules/enabling-debug-mode-clair.adoc @@ -0,0 +1,56 @@ +:_mod-docs-content-type: PROCEDURE +[id="enabling-debug-mode-clair"] +== Enabling debug mode for Clair + +[role="_abstract"] +To enable debug mode for Clair in {productname} and get verbose logging for troubleshooting, you can update your `clair-config.yaml` file to include the `log_level: debug` parameter. + +[NOTE] +==== +For Clair on {productname-ocp} deployments, you must link:https://docs.redhat.com/en/documentation/red_hat_quay/3/html-single/vulnerability_reporting_with_clair_on_red_hat_quay/index#custom-clair-configuration-managed-database[Running a custom Clair configuration with a managed Clair database]. +==== + +.Procedure + +. Update your `clair-config.yaml` file to include the debug option. + +.. On standalone {productname} deployments: + +... Add the following configuration field to your `clair-config.yaml` file: ++ +[source,yaml] +---- +log_level: debug +---- + +... Restart your Clair deployment by entering the following command: ++ +[source,terminal] +---- +$ podman restart +---- + +.. On {productname-ocp} deployments: + +... On the {ocp} web console, click *Operators* -> *Installed Operators* -> *Quay Registry*. + +... Click the name of your registry, for example, *Example Registry*. You are redirected to the *Details* page of your registry. + +... Click the Config Bundle Secret, for example, *example-registry-config-bundle-xncls*. + +... Confirm that you are running a custom Clair configuration by looking for the `clair-config.yaml` file under the *Data* section of the *Details* page of your secret. + +... If you have a `clair-config.yaml` file, click *Actions* -> *Edit Secret*. If you do not, see "Running a custom Clair configuration with a managed Clair database". + +... Update your `clair-config.yaml` file to include the `log_level: debug` configuration variable. For example: ++ +[source,yaml] +---- +log_level: debug +---- + +... Click *Save*. + +... You can check the status of your Clair deployment by clicking *Workloads* -> *Pods*. The `clair-app` pod should report `1/1` under the *Ready* category. + +... You can confirm that Clair is returning debugging information by clicking the *clair-app* pod that is ready -> *Logs*. \ No newline at end of file diff --git a/modules/geo-repl-troubleshooting-issues.adoc b/modules/geo-repl-troubleshooting-issues.adoc index eb2c0a43b..a6bca3fd5 100644 --- a/modules/geo-repl-troubleshooting-issues.adoc +++ b/modules/geo-repl-troubleshooting-issues.adoc @@ -1,82 +1,6 @@ -:_mod-docs-content-type: PROCEDURE +:_mod-docs-content-type: CONCEPT [id="geo-repl-troubleshooting-issues"] = Troubleshooting geo-replication for {productname} -Use the following sections to troubleshoot geo-replication for {productname}. - -//// -[id="check-geo-repl-config"] -== Checking the geo-replication configuration - -Use the following procedure to check your geo-replication configuration in your {productname} `config.yaml` file. - -[IMPORTANT] -==== -The same configuration must be used across all regions. -==== - -.Procedure - -. Check your geo-replication configuration. - -.. If you are using the {productname} Operator, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it quay-pod -- cat /conf/stack/config.yaml ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman exec -it quay-container cat /conf/stack/config.yaml ----- -//// - -[id="check-data-replication"] -== Checking data replication in backend buckets - -Use the following procedure to ensure that your data is properly replicated in all backend buckets. - -.Prerequisites - -* You have installed the `aws` CLI. - -.Procedure - -. Enter the following command to ensure that your data is replicated in all backend buckets: -+ -[source,terminal] ----- -$ aws --profile quay_prod_s3 --endpoint=http://10.0.x.x:port s3 ls ocp-quay --recursive --human-readable --summarize ----- -+ -.Example output -+ -[source,terminal] ----- -Total Objects: 17996 -Total Size: 514.4 GiB ----- - -[id="check-backend-storage-running"] -== Checking the status of your backend storage - -Use the following resources to check the status of your backend storage. - -* *Amazon Web Service Storage (AWS)*. Check the AWS S3 service health status on the link:https://health.aws.amazon.com/health/status[AWS Service Health Dashboard]. Validate your access to S3 by listing objects in a known bucket using the `aws` CLI or SDKs. - -* *Google Cloud Storage (GCS)*. Check the link:https://status.cloud.google.com/[Google Cloud Status Dashboard] for the status of the GCS service. Verify your access to GCS by listing objects in a known bucket using the Google Cloud SDK or GCS client libraries. - -* *NooBaa*. Check the NooBaa management console or administrative interface for any health or status indicators. Ensure that the NooBaa services and related components are running and accessible. Verify access to NooBaa by listing objects in a known bucket using the NooBaa CLI or SDK. - -* **{odf}**. Check the {ocp} Console or management interface for the status of the {odf} components. Verify the availability of {odf} S3 interface and services. Ensure that the {odf} services are running and accessible. Validate access to {odf} S3 by listing objects in a known bucket using the appropriate S3-compatible SDK or CLI. - -* **Ceph**. Check the status of Ceph services, including Ceph monitors, OSDs, and RGWs. Validate that the Ceph cluster is healthy and operational. Verify access to Ceph object storage by listing objects in a known bucket using the appropriate Ceph object storage API or CLI. - -* **Azure Blob Storage**. Check the link:https://azure.status.microsoft/en-us/status[Azure Status Dashboard] to see the health status of the Azure Blob Storage service. Validate your access to Azure Blob Storage by listing containers or objects using the Azure CLI or Azure SDKs. - -* **OpenStack Swift**. Check the link:https://www.ibm.com/docs/ro/cmwo/4.3.0.0?topic=services-checking-status[OpenStack Status] page to verify the status of the OpenStack Swift service. Ensure that the Swift services, like the proxy server, container servers, object servers, are running and accessible. Validate your access to Swift by listing containers or objects using the appropriate Swift CLI or SDK. - -After checking the status of your backend storage, ensure that all {productname} instances have access to all s3 storage backends. \ No newline at end of file +[role="_abstract"] +To troubleshoot geo-replication for {product-title} and resolve replication issues, you can use the procedures in the following sections. These procedures enable you to identify and fix problems with geo-replication deployments. diff --git a/modules/getting-support.adoc b/modules/getting-support.adoc index 42c829761..559b73f5e 100644 --- a/modules/getting-support.adoc +++ b/modules/getting-support.adoc @@ -2,11 +2,8 @@ [id="getting-support"] = Getting support -If you experience difficulty with a procedure described in this documentation, or with {productname} in general, visit the link:http://access.redhat.com[Red Hat Customer Portal]. From the Customer Portal, you can: - -* Search or browse through the Red Hat Knowledgebase of articles and solutions relating to Red Hat products. -* Submit a support case to Red Hat Support. -* Access other product documentation. +[role="_abstract"] +To get help with {product-title} or file a support ticket, you can use the Red Hat Customer Portal and Knowledgebase. You can search articles, submit a support case, or use the debugging tool and health endpoint to gather information before contacting support. To identify issues with your deployment, you can use the {productname} debugging tool, or check the health endpoint of your deployment to obtain information about your problem. After you have debugged or obtained health information about your deployment, you can search the Red Hat Knowledgebase for a solution or file a support ticket. @@ -18,57 +15,4 @@ error, submit a link:https://issues.redhat.com/secure/CreateIssue!default.jspa[J The link:https://access.redhat.com/knowledgebase[Red Hat Knowledgebase] provides rich content aimed at helping you make the most of Red Hat's products and technologies. The Red Hat Knowledgebase consists of articles, product documentation, and videos outlining best practices on installing, configuring, and using Red Hat products. In addition, you can search for solutions to known issues, each providing concise root cause descriptions and remedial steps. -The {productname} Support Team also maintains a link:https://access.redhat.com/articles/6975387[Consolidate troubleshooting article for {productname}] that details solutions to common problems. This is an evolving document that can help users navigate various issues effectively and efficiently. - -[id="support-knowledgebase-search"] -== Searching the Red Hat Knowledgebase - -In the event of an {productname} issue, you can perform an initial search to determine if a solution already exists within the Red Hat Knowledgebase. - -.Prerequisites - -* You have a Red Hat Customer Portal account. - -.Procedure - -. Log in to the link:http://access.redhat.com[Red Hat Customer Portal]. - -. In the main Red Hat Customer Portal search field, input keywords and strings relating to the problem, including: -+ -* {productname} components (such as *database*) -* Related procedure (such as *installation*) -* Warnings, error messages, and other outputs related to explicit failures - -. Click *Search*. - -. Select the *{productname}* product filter. - -. Select the *Knowledgebase* content type filter. - -[id="support-submitting-a-case"] -== Submitting a support case - -.Prerequisites - -* You have a Red Hat Customer Portal account. -* You have a Red Hat standard or premium Subscription. - -.Procedure - -. Log in to the link:http://access.redhat.com[Red Hat Customer Portal] and select *Open a support case*. - -. Select the *Troubleshoot* tab. - -. For *Summary*, enter a concise but descriptive problem summary and further details about the symptoms being experienced, as well as your expectations. - -. Review the list of suggested Red Hat Knowledgebase solutions for a potential match against the problem that is being reported. If the suggested articles do not address the issue, continue to the following step. - -. For *Product*, select *Red Hat Quay*. - -. Select the version of {productname} that you are using. - -. Click *Continue*. - -. Optional. Drag and drop, paste, or browse to upload a file. This could be debug logs gathered from your {productname} deployment. - -. Click *Get support* to file your ticket. \ No newline at end of file +The {productname} Support Team also maintains a link:https://access.redhat.com/articles/6975387[Consolidate troubleshooting article for {productname}] that details solutions to common problems. This is an evolving document that can help users navigate various issues effectively and efficiently. \ No newline at end of file diff --git a/modules/health-check-quay.adoc b/modules/health-check-quay.adoc index 50e08fe7f..e0162239d 100644 --- a/modules/health-check-quay.adoc +++ b/modules/health-check-quay.adoc @@ -2,12 +2,10 @@ [id="health-check-quay"] = Performing health checks on {productname} deployments -Health check mechanisms are designed to assess the health and functionality of a system, service, or component. Health checks help ensure that everything is working correctly, and can be used to identify potential issues before they become critical problems. By monitoring the health of a system, {productname} administrators can address abnormalities or potential failures for things like geo-replication deployments, Operator deployments, standalone {productname} deployments, object storage issues, and so on. Performing health checks can also help reduce the likelihood of encountering troubleshooting scenarios. +[role="_abstract"] +To monitor the health of your {product-title} deployment and identify issues before they become critical, you can perform health checks. Health checks assess system functionality and provide information about the current state for troubleshooting. -Health check mechanisms can play a role in diagnosing issues by providing valuable information about the system's current state. By comparing health check results with expected benchmarks or predefined thresholds, deviations or anomalies can be identified quicker. - -[id="health-check-endpoints"] -== {productname} health check endpoints +Health checks help ensure that everything is working correctly, and can be used to identify potential issues before they become critical problems. By monitoring the health of a system, {productname} administrators can address abnormalities or potential failures for things like geo-replication deployments, Operator deployments, standalone {productname} deployments, object storage issues, and so on. Performing health checks can also help reduce the likelihood of encountering troubleshooting scenarios. Health check mechanisms can play a role in diagnosing issues by providing valuable information about the system's current state. By comparing health check results with expected benchmarks or predefined thresholds, deviations or anomalies can be identified quicker. [IMPORTANT] ==== @@ -26,22 +24,4 @@ Links contained herein to any external website(s) are provided for convenience o |`warning` |The `warning` endpoint conducts a check on the warnings. Returns a `dict` with key-value pairs for the following: `disk_space_warning`. Returns a number indicating the health check response of either `200`, which indicates that the instance is healthy, or `503`, which indicates an issue with your deployment. |`https://{quay-ip-endpoint}/health/warning` | `{"data":{"services":{"disk_space_warning":true}},"status_code":503}` -|=== - -[id="instance-endpoint-quay"] -== Navigating to a {productname} health check endpoint - -Use the following procedure to navigate to the `instance` endpoint. This procedure can be repeated for `endtoend` and `warning` endpoints. - -.Procedure - -. On your web browser, navigate to `https://{quay-ip-endpoint}/health/instance`. - -. You are taken to the health instance page, which returns information like the following: -+ -[source,json] ----- -{"data":{"services":{"auth":true,"database":true,"disk_space":true,"registry_gunicorn":true,"service_key":true,"web_gunicorn":true}},"status_code":200} ----- -+ -For {productname}, `"status_code": 200` means that the instance is health. Conversely, if you receive `"status_code": 503`, there is an issue with your deployment. \ No newline at end of file +|=== \ No newline at end of file diff --git a/modules/instance-endpoint-quay.adoc b/modules/instance-endpoint-quay.adoc new file mode 100644 index 000000000..742d1f511 --- /dev/null +++ b/modules/instance-endpoint-quay.adoc @@ -0,0 +1,19 @@ +:_mod-docs-content-type: PROCEDURE +[id="instance-endpoint-quay"] += Navigating to a {productname} health check endpoint + +[role="_abstract"] +To check the health of your {product-title} instance and view service status, you can navigate to the `health/instance` endpoint in your browser. The endpoint returns JSON with `status_code 200` for healthy or `503` when your deployment has an issue. + +.Procedure + +. On your web browser, navigate to `https://{quay-ip-endpoint}/health/instance`. + +. You are taken to the health instance page, which returns information like the following: ++ +[source,json] +---- +{"data":{"services":{"auth":true,"database":true,"disk_space":true,"registry_gunicorn":true,"service_key":true,"web_gunicorn":true}},"status_code":200} +---- ++ +For {productname}, `"status_code": 200` means that the instance is health. Conversely, if you receive `"status_code": 503`, there is an issue with your deployment. \ No newline at end of file diff --git a/modules/interact-with-database.adoc b/modules/interact-with-database.adoc new file mode 100644 index 000000000..0b7f535b6 --- /dev/null +++ b/modules/interact-with-database.adoc @@ -0,0 +1,55 @@ +:_mod-docs-content-type: PROCEDURE +[id="interact-with-database"] += Interacting with the {productname} database + +[role="_abstract"] +To interact with the {productname} PostgreSQL database and troubleshoot authorization or authentication issues, you can exec into the database using `oc exec` for {productname-ocp} or `podman exec` for standalone, then enter the PostgreSQL shell with `psql`. + +[WARNING] +==== +Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. +==== + +[NOTE] +==== +Interacting with the PostgreSQL database can also be used to troubleshoot authorization and authentication issues. +==== + +.Procedure + +. Exec into the {productname} database. + +.. Enter the following commands to exec into the {productname} database pod on {ocp}: ++ +[source,terminal] +---- +$ oc exec -it -- psql +---- + +.. Enter the following command to exec into the {productname} database on a standalone deployment: ++ +[source,terminal] +---- +$ sudo podman exec -it /bin/bash +---- + +. Enter the PostgreSQL shell. ++ +[WARNING] +==== +Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. +==== + +.. If you are using the {productname} Operator, enter the following command to enter the PostgreSQL shell: ++ +[source,terminal] +---- +$ oc rsh psql -U your_username -d your_database_name +---- + +.. If you are on a standalone {productname} deployment, enter the following command to enter the PostgreSQL shell: ++ +[source,terminal] +---- +bash-4.4$ psql -U your_username -d your_database_name +---- \ No newline at end of file diff --git a/modules/mirroring-intro.adoc b/modules/mirroring-intro.adoc index e70855d19..afb77ad48 100644 --- a/modules/mirroring-intro.adoc +++ b/modules/mirroring-intro.adoc @@ -2,7 +2,8 @@ [id="arch-mirroring-intro"] = Repository mirroring -{productname} repository mirroring lets you mirror images from external container registries, or another local registry, into your {productname} cluster. Using repository mirroring, you can synchronize images to {productname} based on repository names and tags. +[role="_abstract"] +To mirror images from external registries into your {productname} cluster and synchronize by repository names and tags, you can use repository mirroring. From your {productname} cluster with repository mirroring enabled, you can perform the following: diff --git a/modules/obtaining-configuration-information-quay.adoc b/modules/obtaining-configuration-information-quay.adoc new file mode 100644 index 000000000..c2d8a8166 --- /dev/null +++ b/modules/obtaining-configuration-information-quay.adoc @@ -0,0 +1,109 @@ +:_mod-docs-content-type: PROCEDURE +[id="obtaining-configuration-information-quay"] += Obtaining configuration information for {productname} + +[role="_abstract"] +To obtain configuration information for your {product-title} deployment and troubleshoot issues, you can use `oc exec`, `oc cp`, or `oc rsync` for Operator deployments, or `podman cp` or `podman exec` for standalone deployments. You can then update your `config.yaml` file, search the Red Hat Knowledgebase, or file a support ticket. + +.Procedure + +. To obtain configuration information on {productname} Operator deployments, you can use `oc exec`, `oc cp`, or `oc rsync`. + +.. To use the `oc exec` command, enter the following command: ++ +[source,terminal] +---- +$ oc exec -it -- cat /conf/stack/config.yaml +---- ++ +This command returns your `config.yaml` file directly to your terminal. + +.. To use the `oc copy` command, enter the following commands: ++ +[source,terminal] +---- +$ oc cp :/conf/stack/config.yaml /tmp/config.yaml +---- ++ +To display this information in your terminal, enter the following command: ++ +[source,terminal] +---- +$ cat /tmp/config.yaml +---- + +.. To use the `oc rsync` command, enter the following commands: ++ +[source,terminal] +---- +oc rsync :/conf/stack/ /tmp/local_directory/ +---- ++ +To display this information in your terminal, enter the following command: ++ +[source,terminal] +---- +$ cat /tmp/local_directory/config.yaml +---- ++ +.Example output +[source,yaml] +---- +DISTRIBUTED_STORAGE_CONFIG: +local_us: +- RHOCSStorage +- access_key: redacted + bucket_name: lht-quay-datastore-68fff7b8-1b5e-46aa-8110-c4b7ead781f5 + hostname: s3.openshift-storage.svc.cluster.local + is_secure: true + port: 443 + secret_key: redacted + storage_path: /datastorage/registry +DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: +- local_us +DISTRIBUTED_STORAGE_PREFERENCE: +- local_us +---- + +. To obtain configuration information on standalone {productname} deployments, you can use `podman cp` or `podman exec`. + +.. To use the `podman copy` command, enter the following commands: ++ +[source,terminal] +---- +$ podman cp :/conf/stack/config.yaml /tmp/local_directory/ +---- ++ +To display this information in your terminal, enter the following command: ++ +[source,terminal] +---- +$ cat /tmp/local_directory/config.yaml +---- + +.. To use `podman exec`, enter the following commands: ++ +[source,terminal] +---- +$ podman exec -it cat /conf/stack/config.yaml +---- ++ +.Example output +[source,yaml] +---- +BROWSER_API_CALLS_XHR_ONLY: false +ALLOWED_OCI_ARTIFACT_TYPES: + application/vnd.oci.image.config.v1+json: + - application/vnd.oci.image.layer.v1.tar+zstd + application/vnd.sylabs.sif.config.v1+json: + - application/vnd.sylabs.sif.layer.v1+tar +AUTHENTICATION_TYPE: Database +AVATAR_KIND: local +BUILDLOGS_REDIS: + host: quay-server.example.com + password: strongpassword + port: 6379 +DATABASE_SECRET_KEY: 05ee6382-24a6-43c0-b30f-849c8a0f7260 +DB_CONNECTION_ARGS: {} +--- +---- \ No newline at end of file diff --git a/modules/obtaining-db-config-info.adoc b/modules/obtaining-db-config-info.adoc new file mode 100644 index 000000000..752ec3b85 --- /dev/null +++ b/modules/obtaining-db-config-info.adoc @@ -0,0 +1,27 @@ +:_mod-docs-content-type: PROCEDURE +[id="obtaining-db-config-info"] += Obtaining database configuration information + +[role="_abstract"] +To obtain database configuration information for your {product-title} deployment, you can use `oc exec` for Operator deployments or `podman exec` for standalone deployments to read `postgresql.conf` from the database pod or container. + +[WARNING] +==== +Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. +==== + +.Procedure + +* If you are using the {productname} Operator on {ocp}, enter the following command: ++ +[source,terminal] +---- +$ oc exec -it -- cat /var/lib/pgsql/data/userdata/postgresql.conf +---- + +* If you are using a standalone deployment of {productname}, enter the following command: ++ +[source,terminal] +---- +$ podman exec -it cat /var/lib/pgsql/data/userdata/postgresql.conf +---- diff --git a/modules/obtaining-log-information-quay.adoc b/modules/obtaining-log-information-quay.adoc new file mode 100644 index 000000000..c50d06f52 --- /dev/null +++ b/modules/obtaining-log-information-quay.adoc @@ -0,0 +1,30 @@ +:_mod-docs-content-type: PROCEDURE +[id="obtaining-log-information-quay"] += Obtaining log information for {productname} + +[role="_abstract"] +To obtain log information for your {product-title} deployment and troubleshoot authentication, authorization, or object storage issues, you can use oc logs for Operator deployments or podman logs for standalone deployments. You can then search the Red Hat Knowledgebase or file a support ticket. + +.Procedure + +* If you are using the {productname} Operator on {ocp}, enter the following command to view the logs: ++ +[source,terminal] +---- +$ oc logs +---- + +* If you are on a standalone {productname} deployment, enter the following command: ++ +[source,terminal] +---- +$ podman logs +---- ++ +.Example output +[source,terminal] +---- +... +gunicorn-web stdout | 2023-01-20 15:41:52,071 [205] [DEBUG] [app] Starting request: urn:request:0d88de25-03b0-4cf9-b8bc-87f1ac099429 (/oauth2/azure/callback) {'X-Forwarded-For': '174.91.79.124'} +... +---- \ No newline at end of file diff --git a/modules/obtaining-quay-config-information.adoc b/modules/obtaining-quay-config-information.adoc index fd1c65fee..694bb39de 100644 --- a/modules/obtaining-quay-config-information.adoc +++ b/modules/obtaining-quay-config-information.adoc @@ -1,8 +1,11 @@ -:_mod-docs-content-type: PROCEDURE +:_mod-docs-content-type: CONCEPT [id="obtaining-quay-config-information"] = Configuration information for {productname} -Checking a configuration YAML can help identify and resolve various issues related to the configuration of {productname}. Checking the configuration YAML can help you address the following issues: +[role="_abstract"] +To identify and resolve {product-title} configuration issues, you can check the configuration YAML. You can verify parameters, resource limits, connectivity, authentication, replication, and backup settings. + +Checking the configuration YAML can help you address the following issues: * *Incorrect Configuration Parameters*: If the database is not functioning as expected or is experiencing performance issues, your configuration parameters could be at fault. By checking the configuration YAML, administrators can ensure that all the required parameters are set correctly and match the intended settings for the database. @@ -20,140 +23,4 @@ Checking a configuration YAML can help identify and resolve various issues relat * *Backup and Recovery Options*: The configuration YAML might include backup and recovery options, specifying how data backups are performed and how data can be recovered in case of failures. Validating these settings can ensure data safety and successful recovery processes. -By checking your configuration YAML, {productname} administrators can detect and resolve these issues before they cause significant disruptions to the application or service relying on the database. - -[id="obtaining-configuration-information-quay"] -== Obtaining configuration information for {productname} - -Configuration information can be obtained for all types of {productname} deployments, include standalone, Operator, and geo-replication deployments. Obtaining configuration information can help you resolve issues with authentication and authorization, your database, object storage, and repository mirroring. After you have obtained the necessary configuration information, you can update your `config.yaml` file, search the link:https://access.redhat.com/knowledgebase[Red Hat Knowledgebase] for a solution, or file a support ticket with the Red Hat Support team. - -.Procedure - -. To obtain configuration information on {productname} Operator deployments, you can use `oc exec`, `oc cp`, or `oc rsync`. - -.. To use the `oc exec` command, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it -- cat /conf/stack/config.yaml ----- -+ -This command returns your `config.yaml` file directly to your terminal. - -.. To use the `oc copy` command, enter the following commands: -+ -[source,terminal] ----- -$ oc cp :/conf/stack/config.yaml /tmp/config.yaml ----- -+ -To display this information in your terminal, enter the following command: -+ -[source,terminal] ----- -$ cat /tmp/config.yaml ----- - -.. To use the `oc rsync` command, enter the following commands: -+ -[source,terminal] ----- -oc rsync :/conf/stack/ /tmp/local_directory/ ----- -+ -To display this information in your terminal, enter the following command: -+ -[source,terminal] ----- -$ cat /tmp/local_directory/config.yaml ----- -+ -.Example output -+ -[source,yaml] ----- -DISTRIBUTED_STORAGE_CONFIG: -local_us: -- RHOCSStorage -- access_key: redacted - bucket_name: lht-quay-datastore-68fff7b8-1b5e-46aa-8110-c4b7ead781f5 - hostname: s3.openshift-storage.svc.cluster.local - is_secure: true - port: 443 - secret_key: redacted - storage_path: /datastorage/registry -DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: -- local_us -DISTRIBUTED_STORAGE_PREFERENCE: -- local_us ----- - -. To obtain configuration information on standalone {productname} deployments, you can use `podman cp` or `podman exec`. - -.. To use the `podman copy` command, enter the following commands: -+ -[source,terminal] ----- -$ podman cp :/conf/stack/config.yaml /tmp/local_directory/ ----- -+ -To display this information in your terminal, enter the following command: -+ -[source,terminal] ----- -$ cat /tmp/local_directory/config.yaml ----- - -.. To use `podman exec`, enter the following commands: -+ -[source,terminal] ----- -$ podman exec -it cat /conf/stack/config.yaml ----- -+ -.Example output -+ -[source,yaml] ----- -BROWSER_API_CALLS_XHR_ONLY: false -ALLOWED_OCI_ARTIFACT_TYPES: - application/vnd.oci.image.config.v1+json: - - application/vnd.oci.image.layer.v1.tar+zstd - application/vnd.sylabs.sif.config.v1+json: - - application/vnd.sylabs.sif.layer.v1+tar -AUTHENTICATION_TYPE: Database -AVATAR_KIND: local -BUILDLOGS_REDIS: - host: quay-server.example.com - password: strongpassword - port: 6379 -DATABASE_SECRET_KEY: 05ee6382-24a6-43c0-b30f-849c8a0f7260 -DB_CONNECTION_ARGS: {} ---- ----- - -[id="obtaining-db-config-info"] -== Obtaining database configuration information - -You can obtain configuration information about your database by using the following procedure. - -[WARNING] -==== -Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. -==== - -.Procedure - -* If you are using the {productname} Operator on {ocp}, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it -- cat /var/lib/pgsql/data/userdata/postgresql.conf ----- - -* If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman exec -it cat /var/lib/pgsql/data/userdata/postgresql.conf ----- +By checking your configuration YAML, {productname} administrators can detect and resolve these issues before they cause significant disruptions to the application or service relying on the database. \ No newline at end of file diff --git a/modules/obtaining-quay-logs.adoc b/modules/obtaining-quay-logs.adoc index e911ba512..044f82f33 100644 --- a/modules/obtaining-quay-logs.adoc +++ b/modules/obtaining-quay-logs.adoc @@ -1,8 +1,11 @@ -:_mod-docs-content-type: PROCEDURE +:_mod-docs-content-type: CONCEPT [id="obtaining-quay-logs"] = Logging information for {productname} -Obtaining log information using can be beneficial in various ways for managing, monitoring, and troubleshooting applications running in containers or pods. Some of the reasons why obtaining log information is valuable include the following: +[role="_abstract"] +To troubleshoot, monitor, and secure your {product-title} deployment, you can use log information from containers and pods. Logs support debugging, performance monitoring, security analysis, capacity planning, and deployment verification. + +Some of the reasons why obtaining log information is valuable include the following: * *Debugging and Troubleshooting*: Logs provide insights into what's happening inside the application, allowing developers and system administrators to identify and resolve issues. By analyzing log messages, one can identify errors, exceptions, warnings, or unexpected behavior that might occur during the application's execution. @@ -18,89 +21,4 @@ Obtaining log information using can be beneficial in various ways for managing, * *Verification of Deployment*: Logging during the deployment process can help verify if the application is starting correctly and if all components are functioning as expected. -* *Continuous Integration/Continuous Deployment (CI/CD)*: In CI/CD pipelines, logging is essential to capture build and deployment statuses, allowing teams to monitor the success or failure of each stage. - -[id="obtaining-log-information-quay"] -== Obtaining log information for {productname} - -Log information can be obtained for all types of {productname} deployments, including geo-replication deployments, standalone deployments, and Operator deployments. Log information can also be obtained for mirrored repositories. It can help you troubleshoot authentication and authorization issues, and object storage issues. After you have obtained the necessary log information, you can search the link:https://access.redhat.com/knowledgebase[Red Hat Knowledgebase] for a solution, or file a support ticket with the Red Hat Support team. - -Use the following procedure to obtain logs for your {productname} deployment. - -.Procedure - -* If you are using the {productname} Operator on {ocp}, enter the following command to view the logs: -+ -[source,terminal] ----- -$ oc logs ----- - -* If you are on a standalone {productname} deployment, enter the following command: -+ -[source,terminal] ----- -$ podman logs ----- -+ -.Example output -+ -[source,terminal] ----- -... -gunicorn-web stdout | 2023-01-20 15:41:52,071 [205] [DEBUG] [app] Starting request: urn:request:0d88de25-03b0-4cf9-b8bc-87f1ac099429 (/oauth2/azure/callback) {'X-Forwarded-For': '174.91.79.124'} -... ----- - -[id="obtaining-verbose-container-pod-logs"] -== Examining verbose logs - -{productname} does not have verbose logs, however, with the following procedures, you can obtain a detailed status check of your database pod or container. - -[NOTE] -==== -Additional debugging information can be returned if you have deployed {productname} in one of the following ways: - -* You have deployed {productname} by passing in the `DEBUGLOG=true` variable. -* You have deployed {productname} with LDAP authentication enabled by passing in the `DEBUGLOG=true` and `USERS_DEBUG=1` variables. -* You have configured {productname-ocp} by updating the `QuayRegistry` resource to include `DEBUGLOG=true`. - -For more information, see "Running {productname} in debug mode". -==== -.Procedure - -. Enter the following commands to examine verbose database logs. - -.. If you are using the {productname} Operator on {ocp}, enter the following commands: -+ -[source,terminal] ----- -$ oc logs --previous ----- -+ -[source,terminal] ----- -$ oc logs --previous -c ----- -+ -[source,terminal] ----- -$ oc cp :/var/lib/pgsql/data/userdata/log/* /path/to/desired_directory_on_host ----- - -.. If you are using a standalone deployment of {productname}, enter the following commands: -+ -[source,terminal] ----- -$ podman logs --previous ----- -+ -[source,terminal] ----- -$ podman logs --previous -c ----- -+ -[source,terminal] ----- -$ podman cp :/var/lib/pgsql/data/userdata/log/* /path/to/desired_directory_on_host ----- \ No newline at end of file +* *Continuous Integration/Continuous Deployment (CI/CD)*: In CI/CD pipelines, logging is essential to capture build and deployment statuses, allowing teams to monitor the success or failure of each stage. \ No newline at end of file diff --git a/modules/obtaining-verbose-container-pod-logs.adoc b/modules/obtaining-verbose-container-pod-logs.adoc new file mode 100644 index 000000000..7f2617050 --- /dev/null +++ b/modules/obtaining-verbose-container-pod-logs.adoc @@ -0,0 +1,55 @@ +:_mod-docs-content-type: PROCEDURE +[id="obtaining-verbose-container-pod-logs"] += Examining verbose logs + +[role="_abstract"] +To get a detailed status check of your {product-title} database pod or container, you can use oc logs or podman logs with --previous and copy PostgreSQL logs with oc cp or podman cp. Enable DEBUGLOG=true for additional debugging information. + +[NOTE] +==== +Additional debugging information can be returned if you have deployed {productname} in one of the following ways: + +* You have deployed {productname} by passing in the `DEBUGLOG=true` variable. +* You have deployed {productname} with LDAP authentication enabled by passing in the `DEBUGLOG=true` and `USERS_DEBUG=1` variables. +* You have configured {productname-ocp} by updating the `QuayRegistry` resource to include `DEBUGLOG=true`. + +For more information, see "Running {productname} in debug mode". +==== + +.Procedure + +. Enter the following commands to examine verbose database logs. + +.. If you are using the {productname} Operator on {ocp}, enter the following commands: ++ +[source,terminal] +---- +$ oc logs --previous +---- ++ +[source,terminal] +---- +$ oc logs --previous -c +---- ++ +[source,terminal] +---- +$ oc cp :/var/lib/pgsql/data/userdata/log/* /path/to/desired_directory_on_host +---- + +.. If you are using a standalone deployment of {productname}, enter the following commands: ++ +[source,terminal] +---- +$ podman logs --previous +---- ++ +[source,terminal] +---- +$ podman logs --previous -c +---- ++ +[source,terminal] +---- +$ podman cp :/var/lib/pgsql/data/userdata/log/* /path/to/desired_directory_on_host +---- \ No newline at end of file diff --git a/modules/repo-mirroring-troubleshooting-issues.adoc b/modules/repo-mirroring-troubleshooting-issues.adoc index 5be7d748e..5491298f4 100644 --- a/modules/repo-mirroring-troubleshooting-issues.adoc +++ b/modules/repo-mirroring-troubleshooting-issues.adoc @@ -1,112 +1,12 @@ -:_mod-docs-content-type: PROCEDURE +:_mod-docs-content-type: CONCEPT [id="repo-mirroring-troubleshooting-issues"] -= Troubleshooting repository mirroring += Verifying authentication and permissions -Use the following sections to troubleshoot repository mirroring for {productname}. - -//// -[id="reviewing-logs-repo-mirroring"] -== Reviewing the logs of your mirrored {productname} instances - -Use the following procedure to review the logs of your mirrored instances. - -.Prerequisites - -* You have enabled debug mode in your {productname} `config.yaml` file. - -.Procedure - -* Retrieve the logs from all running mirror pods. - -.. If you are using the {productname} Operator, enter the following command: -+ -[source,terminal] ----- -$ oc logs mirror-pod ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman logs repomirror-container ----- - - -[id="checking-mirror-configuration"] -== Checking the mirror configuration - -Use the following procedure to review the mirror configuration settings in your {productname} instances. - -.Procedure - -* Review your `config.yaml` settings. - -.. If you are using the {productname} Operator, enter the following command: -+ -[source,terminal] ----- -$ oc exec -it quay-pod -- cat /conf/stack/config.yaml ----- - -.. If you are using a standalone deployment of {productname}, enter the following command: -+ -[source,terminal] ----- -$ podman exec -it quay-container cat /conf/stack/config.yaml ----- -//// - -[id="verifying-authentication-permissions"] -== Verifying authentication and permissions - -Ensure that the authentication credentials used for mirroring have the necessary permissions and access rights on both the source and destination {productname} instances. +[role="_abstract"] +To verify authentication and permissions for {product-title} repository mirroring and resolve mirroring issues, you can ensure credentials have the necessary access on source and destination instances. On the {productname} UI, check the following settings: * The access control settings. Ensure that the user or service account performing the mirroring operation has the required privileges. -* The permissions of your robot account on the {productname} registry. - -//// -[id="manual-copy"] -== Checking slow disk issues - -Repository mirroring uses `skopeo copy` as a background process. Test the time it takes to copy an image by manually running `skopeo copy`. This can help isolate any issues related to specific images or repositories and narrow down the troubleshooting scope. Additionally, it can help identify any network issues or bottlenecks that might be impacting the mirroring performance or causing failures. Pay attention to network latency, packet loss, or any unusual network patterns. -Use the following procedure to time `skopeo copy`. - -.Procedure - -* Enter the following command to measure the time it takes to perform `skopeo copy`: -+ -[source,terminal] ----- -$ time { skopeo copy docker://SOURCE_REGISTRY_IMAGE docker://DESTINATION_REGISTRY/REPOSITPRY/IMAGE:TAG } ----- -+ -.Example output -+ -[source,terminal] ----- -Getting image source signatures -Copying blob 4182b7568f06 skipped: already exists -Copying blob 4182b7568f06 skipped: already exists -Copying blob b7f76d1d9088 skipped: already exists -Copying blob ede3648667b7 skipped: already exists -Copying blob 021495d3c262 done -Copying blob 335fbccacdd3 done -Copying blob 4c70e3d931b6 done -Copying config d9f6ca2777 done -Writing manifest to image destination -Storing signatures - -real 6m19.291s -user 0m58.207s -sys 0m40.666s ----- - -[role="_additional-resources"] -.Additional resources - -For more information, see link:https://access.redhat.com/articles/7018078[Troubleshooting Quay Repository Mirroring]. -//// \ No newline at end of file +* The permissions of your robot account on the {productname} registry. diff --git a/modules/resetting-superuser-password-on-operator.adoc b/modules/resetting-superuser-password-on-operator.adoc index 59d5e0370..df7dee323 100644 --- a/modules/resetting-superuser-password-on-operator.adoc +++ b/modules/resetting-superuser-password-on-operator.adoc @@ -1,7 +1,10 @@ -:_mod-docs-content-type: CONCEPT +:_mod-docs-content-type: PROCEDURE [id="resetting-superuser-password-on-operator"] = Resetting superuser passwords on the {productname} Operator +[role="_abstract"] +To reset the superuser password on your {productname} Operator deployment, you can generate a bcrypt-hashed password with Python, log in to the PostgreSQL database with `oc rsh` and `psql`, and update the user table with the new hashed password. + .Prerequisites * You have created a {productname} superuser. @@ -27,7 +30,6 @@ $ python3.9 -c 'import bcrypt; print(bcrypt.hashpw(b"newpass1234", bcrypt.gensal ---- + .Example output -+ [source,terminal] ---- $2b$12$zoilcTG6XQeAoVuDuIZH0..UpvQEZcKh3V6puksQJaUQupHgJ4.4y @@ -69,7 +71,6 @@ quay=> select * from public.user; ---- + .Example output -+ [source,terminal] ---- id | uuid | username | password_hash | email | verified | stripe_id | organization | robot | invoice_email | invalid_login_attempts | last_invalid_login |removed_tag_expiration_s | enabled | invoice_email_address | company | family_name | given_name | location | maximum_queued_builds_count | creation_date | last_accessed diff --git a/modules/running-ldap-debug-mode.adoc b/modules/running-ldap-debug-mode.adoc index 2a9aeb07f..35823a5fc 100644 --- a/modules/running-ldap-debug-mode.adoc +++ b/modules/running-ldap-debug-mode.adoc @@ -2,7 +2,8 @@ [id="running-ldap-debug-mode"] = Running an LDAP {productname} deployment in debug mode -Use the following procedure to run an LDAP deployment of {productname} in debug mode. +[role="_abstract"] +To run an LDAP {productname} deployment in debug mode and debug LDAP operations, you can use `podman run` with `DEBUGLOG=true` and `USERS_DEBUG=1`. View debug output with the `podman logs` command. .Procedure diff --git a/modules/running-operator-debug-mode.adoc b/modules/running-operator-debug-mode.adoc index 2cc15dec7..28d085f18 100644 --- a/modules/running-operator-debug-mode.adoc +++ b/modules/running-operator-debug-mode.adoc @@ -2,7 +2,8 @@ [id="running-operator-debug-mode"] = Running the {productname} Operator in debug mode -Use the following procedure to run the {productname} Operator in debug mode. +[role="_abstract"] +To run the {product-title} Operator in debug mode and get verbose logging for troubleshooting, you can edit the `QuayRegistry` custom resource and add `DEBUGLOG=true` to the env overrides. After the Operator restarts, try pulling an image or dump logs from Quay pods for more information. .Procedure diff --git a/modules/running-quay-debug-mode-intro.adoc b/modules/running-quay-debug-mode-intro.adoc index 39cca0844..641fa1a2e 100644 --- a/modules/running-quay-debug-mode-intro.adoc +++ b/modules/running-quay-debug-mode-intro.adoc @@ -2,7 +2,8 @@ [id="running-quay-debug-mode-intro"] = Running {productname} in debug mode -Red Hat recommends gathering your debugging information when opening a support case. Running {productname} in debug mode provides verbose logging to help administrators find more information about various issues. Enabling debug mode can speed up the process to reproduce errors and validate a solution for things like geo-replication deployments, Operator deployments, standalone {productname} deployments, object storage issues, and so on. Additionally, it helps the Red Hat Support to perform a root cause analysis. +[role="_abstract"] +To gather debugging information for support cases and troubleshoot issues, you can run {product-title} in debug mode. Debug mode provides verbose logging that speeds up reproducing errors and supports root cause analysis for geo-replication, Operator, standalone deployments, and object storage. [id="debug-configuration-fields"] == {productname} debug variables @@ -10,11 +11,12 @@ Red Hat recommends gathering your debugging information when opening a support c {productname} offers two configuration fields that can be added to your `config.yaml` file to help diagnose issues or help obtain log information. .Debug configuration variables -[cols="3a,1a,2a",options="header"] +[cols="1a,1a,3a",options="header"] |=== | Variable | Type | Description | **DEBUGLOG** | Boolean | Whether to enable or disable debug logs. Must be `True` or `False`. -| **USERS_DEBUG** |Integer. Either `0` or `1`. | Used to debug LDAP operations in clear text, including passwords. Must be used with `DEBUGLOG=TRUE`. + +| **USERS_DEBUG** |Integer. Either `0` or `1`. | Used to debug LDAP operations in clear text, including passwords. Must be used with `DEBUGLOG=TRUE`. + [IMPORTANT] ==== Setting `USERS_DEBUG=1` exposes credentials in clear text. This variable should be removed from the {productname} deployment after debugging. The log file that is generated with this environment variable should be scrutinized, and passwords should be removed before sending to other users. Use with caution. diff --git a/modules/running-quay-debug-mode.adoc b/modules/running-quay-debug-mode.adoc index 0fc24b52c..6067cf2f5 100644 --- a/modules/running-quay-debug-mode.adoc +++ b/modules/running-quay-debug-mode.adoc @@ -2,9 +2,8 @@ [id="running-standalone-debug-mode"] = Running a standalone {productname} deployment in debug mode -Running {productname} in debug mode provides verbose logging to help administrators find more information about various issues. Enabling debug mode can speed up the process to reproduce errors and validate a solution. - -Use the following procedure to run a standalone deployment of {productname} in debug mode. +[role="_abstract"] +To run a standalone {productname} deployment in debug mode and get verbose logging for troubleshooting, you can use `podman run` with `DEBUGLOG=true`. View debug output with the `podman logs` command. .Procedure diff --git a/modules/storage-troubleshooting-issues.adoc b/modules/storage-troubleshooting-issues.adoc index 784f6b515..f8cc0b6ff 100644 --- a/modules/storage-troubleshooting-issues.adoc +++ b/modules/storage-troubleshooting-issues.adoc @@ -2,7 +2,8 @@ [id="storage-troubleshooting-issues"] = Troubleshooting {productname} object storage issues -Use the following options to troubleshoot {productname} object storage issues. +[role="_abstract"] +To troubleshoot {productname} object storage issues, you can check the `QuayRegistry` CR and `config.yaml` file, verify supported storage and network connectivity, enable debug mode, and test storage access outside Quay. .Procedure diff --git a/modules/storage-troubleshooting.adoc b/modules/storage-troubleshooting.adoc index 7c0a317b9..ec77fe759 100644 --- a/modules/storage-troubleshooting.adoc +++ b/modules/storage-troubleshooting.adoc @@ -2,6 +2,9 @@ [id="storage-troubleshooting"] = Troubleshooting {productname} object storage +[role="_abstract"] +To troubleshoot {productname} object storage and resolve issues with container image storage, you can use the procedures in this section. Object storage manages data as discrete units called objects, each with a unique identifier and metadata. + Object storage is a type of data storage architecture that manages data as discrete units called `objects`. Unlike traditional file systems that organize data into hierarchical directories and files, object storage treats data as independent entities with unique identifiers. Each object contains the data itself, along with metadata that describes the object and enables efficient retrieval. {productname} uses object storage as the underlying storage mechanism for storing and managing container images. It stores container images as individual objects. Each container image is treated as an object, with its own unique identifier and associated metadata. \ No newline at end of file diff --git a/modules/support-knowledgebase-search.adoc b/modules/support-knowledgebase-search.adoc new file mode 100644 index 000000000..1f8043045 --- /dev/null +++ b/modules/support-knowledgebase-search.adoc @@ -0,0 +1,26 @@ +:_mod-docs-content-type: PROCEDURE +[id="support-knowledgebase-search"] += Searching the Red Hat Knowledgebase + +[role="_abstract"] +To find solutions for {product-title} issues, you can search the Red Hat Knowledgebase. Log in to the Customer Portal, enter keywords related to your problem, and apply the {product-title} and Knowledgebase filters. + +.Prerequisites + +* You have a Red Hat Customer Portal account. + +.Procedure + +. Log in to the link:http://access.redhat.com[Red Hat Customer Portal]. + +. In the main Red Hat Customer Portal search field, input keywords and strings relating to the problem, including: ++ +* {productname} components (such as *database*) +* Related procedure (such as *installation*) +* Warnings, error messages, and other outputs related to explicit failures + +. Click *Search*. + +. Select the *{productname}* product filter. + +. Select the *Knowledgebase* content type filter. \ No newline at end of file diff --git a/modules/support-submitting-a-case.adoc b/modules/support-submitting-a-case.adoc new file mode 100644 index 000000000..aeb3c220d --- /dev/null +++ b/modules/support-submitting-a-case.adoc @@ -0,0 +1,31 @@ +:_mod-docs-content-type: PROCEDURE +[id="support-submitting-a-case"] += Submitting a support case + +[role="_abstract"] +To file a support ticket for {product-title}, you can submit a support case from the Red Hat Customer Portal. Log in, open a support case, enter problem details, select {productname} and your version, and optionally attach debug logs. + +.Prerequisites + +* You have a Red Hat Customer Portal account. +* You have a Red Hat standard or premium Subscription. + +.Procedure + +. Log in to the link:http://access.redhat.com[Red Hat Customer Portal] and select *Open a support case*. + +. Select the *Troubleshoot* tab. + +. For *Summary*, enter a concise but descriptive problem summary and further details about the symptoms being experienced, as well as your expectations. + +. Review the list of suggested Red Hat Knowledgebase solutions for a potential match against the problem that is being reported. If the suggested articles do not address the issue, continue to the following step. + +. For *Product*, select *{productname}*. + +. Select the version of {productname} that you are using. + +. Click *Continue*. + +. Optional. Drag and drop, paste, or browse to upload a file. This could be debug logs gathered from your {productname} deployment. + +. Click *Get support* to file your ticket. \ No newline at end of file diff --git a/modules/troubleshooting-components.adoc b/modules/troubleshooting-components.adoc index 04ef36d05..e30dd6828 100644 --- a/modules/troubleshooting-components.adoc +++ b/modules/troubleshooting-components.adoc @@ -2,8 +2,7 @@ [id="troubleshooting-components"] = Troubleshooting {productname} components -This document focuses on troubleshooting specific components within {productname}, providing targeted guidance for resolving issues that might arise. Designed for system administrators, operators, and developers, this resource aims to help diagnose and troubleshoot problems related to individual components of {productname}. +[role="_abstract"] +To troubleshoot specific {product-title} components and resolve component-related issues, you can use the procedures in this document. You can also run in debug mode, obtain logs and config, perform health checks, then search the Red Hat Knowledgebase or file a support ticket. -In addition to the following procedures, {productname} components can also be troubleshot by running {productname} in debug mode, obtaining log information, obtaining configuration information, and performing health checks on endpoints. - -By using the following procedures, you are able to troubleshoot common component issues. Afterwards, you can search for solutions on the link:https://access.redhat.com/knowledgebase[Red Hat Knowledgebase], or file a support ticket with the Red Hat Support team. \ No newline at end of file +By using the following procedures, you are able to troubleshoot common component issues. Afterwards, you can search for solutions on the Red Hat Knowledgebase or file a support ticket with the Red Hat Support team. \ No newline at end of file diff --git a/modules/troubleshooting-crashloop-backoff-state.adoc b/modules/troubleshooting-crashloop-backoff-state.adoc new file mode 100644 index 000000000..03cb97d2a --- /dev/null +++ b/modules/troubleshooting-crashloop-backoff-state.adoc @@ -0,0 +1,92 @@ +:_mod-docs-content-type: PROCEDURE +[id="troubleshooting-crashloop-backoff-state"] += Troubleshooting crashloopbackoff states + +[role="_abstract"] +To troubleshoot `crashloopbackoff` states for your {productname} deployment and restore pods or containers, you can scale down the Quay Operator and database, then edit the database deployment as needed. + +.Procedure + +. If your container or pod is in a `crashloopbackoff` state, you can enter the following commands. + +.. Enter the following command to scale down the {productname} Operator: ++ +[source,terminal] +---- +$ oc scale deployment/quay-operator.v3.8.z --replicas=0 +---- ++ +.Example output +[source,terminal] +---- +deployment.apps/quay-operator.v3.8.z scaled +---- + +.. Enter the following command to scale down the {productname} database: ++ +[source,terminal] +---- +$ oc scale deployment/ --replicas=0 +---- ++ +.Example output +[source,terminal] +---- +deployment.apps/ scaled +---- + +.. Enter the following command to edit the {productname} database: ++ +[WARNING] +==== +Interacting with the PostgreSQL database is potentially destructive. It is highly recommended that you perform the following procedure with the help of a {productname} Support Specialist. +==== ++ +[source,terminal] +---- +$ oc edit deployment +---- ++ +[source,yaml] +---- +... + template: + metadata: + creationTimestamp: null + labels: + quay-component: + quay-operator/quayregistry: quay-operator.v3.8.z + spec: + containers: + - env: + - name: POSTGRESQL_USER + value: postgres + - name: POSTGRESQL_DATABASE + value: postgres + - name: POSTGRESQL_PASSWORD + value: postgres + - name: POSTGRESQL_ADMIN_PASSWORD + value: postgres + - name: POSTGRESQL_MAX_CONNECTIONS + value: "1000" + image: registry.redhat.io/rhel8/postgresql-10@sha256:a52ad402458ec8ef3f275972c6ebed05ad64398f884404b9bb8e3010c5c95291 + imagePullPolicy: IfNotPresent + name: postgres + command: ["/bin/bash", "-c", "sleep 86400"] <1> +... +---- ++ +** Add the `command: ["/bin/bash", "-c", "sleep 86400"]` line in the same indentation. ++ +.Example output +[source,terminal] +---- +deployment.apps/ edited +---- + +.. Execute the following command inside of your ``: ++ +[source,terminal] +---- +$ oc exec -it -- cat /var/lib/pgsql/data/userdata/postgresql/logs/* /path/to/desired_directory_on_host +---- \ No newline at end of file diff --git a/modules/troubleshooting-forgotten-passwords.adoc b/modules/troubleshooting-forgotten-passwords.adoc index 1ae48fb43..937eecdf6 100644 --- a/modules/troubleshooting-forgotten-passwords.adoc +++ b/modules/troubleshooting-forgotten-passwords.adoc @@ -1,8 +1,9 @@ -:_mod-docs-content-type: CONCEPT +:_mod-docs-content-type: PROCEDURE [id="troubleshooting-forgotten-passwords"] = Resetting superuser passwords on {productname} standalone deployments -Use the following procedure to reset a superuser's password. +[role="_abstract"] +To reset the superuser password on your {productname} standalone deployment, you can generate a bcrypt-hashed password with Python, exec into the PostgreSQL container with `podman exec`, and update the user table with the new hashed password. .Prerequisites @@ -21,7 +22,6 @@ $ python3.9 -c 'import bcrypt; print(bcrypt.hashpw(b"newpass1234", bcrypt.gensal ---- + .Example output -+ [source,terminal] ---- $2b$12$T8pkgtOoys3G5ut7FV1She6vXlYgU.6TeoGmbbAVQtN8X8ch4knKm @@ -35,7 +35,6 @@ $ sudo podman ps -a ---- + .Example output -+ [source,terminal] ---- CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES @@ -66,7 +65,6 @@ quay=> UPDATE public.user SET password_hash = '$2b$12$T8pkgtOoys3G5ut7FV1She6vXl ---- + .Example output -+ [source,terminal] ---- UPDATE 1 @@ -80,7 +78,6 @@ quay=> select * from public.user; ---- + .Example output -+ [source,terminal] ---- id | uuid | username | password_hash | email | verified | stripe_id | organization | robot | invoice_email | invalid_login_attempts | last_invalid_login |removed_tag_expiration_s | enabled | invoice_email_address | company | family_name | given_name | location | maximum_queued_builds_count | creation_date | last_accessed @@ -98,13 +95,7 @@ $ sudo podman login -u quayadmin -p newpass1234 http://quay-server.example.com - ---- + .Example output -+ [source,terminal] ---- Login Succeeded! ---- - -[role="_additional-resources"] -.Additional resources - -For more information, see link:https://access.redhat.com/solutions/6964805[Resetting Superuser Password for Quay]. diff --git a/troubleshooting_quay/master.adoc b/troubleshooting_quay/master.adoc index 1c8432f02..800dd7de1 100644 --- a/troubleshooting_quay/master.adoc +++ b/troubleshooting_quay/master.adoc @@ -1,11 +1,16 @@ -include::_attributes/attributes.adoc[]:_mod-docs-content-type: ASSEMBLY +:_mod-docs-content-type: ASSEMBLY +include::_attributes/attributes.adoc[] [id="support-overview"] = Troubleshooting {productname} +:context: troubleshooting-quay -Red Hat offers administrators tools for gathering data for your {productname} deployment. You can use this data to troubleshoot your {productname} deployment yourself, or file a support ticket. +[role="_abstract"] +To troubleshoot your {product-title} deployment and resolve issues yourself or prepare a support ticket, you can use this guide. You can gather logs and config, run in debug mode, perform health checks, and fix common errors on standalone and Operator-based deployments. //Support include::modules/getting-support.adoc[leveloffset=+1] +include::modules/support-knowledgebase-search.adoc[leveloffset=+2] +include::modules/support-submitting-a-case.adoc[leveloffset=+2] //Debug mode include::modules/running-quay-debug-mode-intro.adoc[leveloffset=+1] @@ -15,12 +20,17 @@ include::modules/running-operator-debug-mode.adoc[leveloffset=+2] //quay logs include::modules/obtaining-quay-logs.adoc[leveloffset=+1] +include::modules/obtaining-log-information-quay.adoc[leveloffset=+2] +include::modules/obtaining-verbose-container-pod-logs.adoc[leveloffset=+2] //quay config include::modules/obtaining-quay-config-information.adoc[leveloffset=+1] +include::modules/obtaining-configuration-information-quay.adoc[leveloffset=+2] +include::modules/obtaining-db-config-info.adoc[leveloffset=+2] //health-check include::modules/health-check-quay.adoc[leveloffset=+1] +include::modules/instance-endpoint-quay.adoc[leveloffset=+2] //Troubleshooting components include::modules/troubleshooting-components.adoc[leveloffset=+1] @@ -30,85 +40,41 @@ include::modules/database-troubleshooting-issues.adoc[leveloffset=+3] include::modules/troubleshooting-forgotten-passwords.adoc[leveloffset=+3] include::modules/resetting-superuser-password-on-operator.adoc[leveloffset=+3] +[id="additional-resources_superuser-password"] +[role="_additional-resources"] +.Additional resources + +* link:https://access.redhat.com/solutions/6964805[Resetting Superuser Password for Quay] + // Authentication include::modules/authentication-troubleshooting.adoc[leveloffset=+2] include::modules/authentication-troubleshooting-issues.adoc[leveloffset=+3] +include::modules/interact-with-database.adoc[leveloffset=+3] +include::modules/troubleshooting-crashloop-backoff-state.adoc[leveloffset=+3] +include::modules/connectivity-networking.adoc[leveloffset=+3] +include::modules/check-resource-allocation.adoc[leveloffset=+3] //Storage include::modules/storage-troubleshooting.adoc[leveloffset=+2] include::modules/storage-troubleshooting-issues.adoc[leveloffset=+3] -//include::modules/changing-storage-solution.adoc[leveloffset=+3] -//include::modules/connecting-s3-timeout.adoc[leveloffset=+3] + //Geo replication -include::modules/georepl-intro.adoc[leveloffset=+2] -include::modules/geo-repl-troubleshooting-issues.adoc[leveloffset=+3] -//include::modules/storage-health-check-geo-repl.adoc[leveloffset=+3] -//include::modules/storage-buckets-not-synced.adoc[leveloffset=+3] -//include::modules/geo-repl-sslerror.adoc[leveloffset=+3] +include::modules/georepl-intro.adoc[leveloffset=+1] +include::modules/geo-repl-troubleshooting-issues.adoc[leveloffset=+2] +include::modules/check-data-replication.adoc[leveloffset=+3] +include::modules/check-backend-storage-running.adoc[leveloffset=+3] //Repository mirroring -include::modules/mirroring-intro.adoc[leveloffset=+2] -include::modules/repo-mirroring-troubleshooting-issues.adoc[leveloffset=+3] -//include::modules/mirroring-invalid-credentials.adoc[leveloffset=+3] -//include::modules/missing-runc-files.adoc[leveloffset=+3] -//include::modules/signature-does-not-exist.adoc[leveloffset=+3] - +include::modules/mirroring-intro.adoc[leveloffset=+1] +include::modules/repo-mirroring-troubleshooting-issues.adoc[leveloffset=+2] //Clair include::modules/clair-vulnerability-scanner-overview.adoc[leveloffset=+2] -//include::modules/clair-concepts.adoc[leveloffset=+3] include::modules/clair-troubleshooting-issues.adoc[leveloffset=+3] -//include::modules/unsupported-security-scan.adoc[leveloffset=+3] -//include::modules/scans-not-working-behind-proxy.adoc[leveloffset=+3] -//include::modules/connection-issues-clair-quay-db.adoc[leveloffset=+3] -//include::modules/java-image-scan-not-working.adoc[leveloffset=+3] +[id="additional-resources_clair"] +[role="_additional-resources"] +.Additional resources -//// - - -[id="troubleshooting-quay"] -= Troubleshooting {productname} - -Use the content in this guide to troubleshoot your {productname} registry on both standalone and Operator based deployments. - - -//General Troubleshooting -include::modules/troubleshooting-general.adoc[leveloffset=+1] -include::modules/troubleshooting-401-helm.adoc[leveloffset=+2] -include::modules/error-403-troubleshooting.adoc[leveloffset=+2] -include::modules/error-406-dockerfile.adoc[leveloffset=+2] -include::modules/error-429-troubleshooting.adoc[leveloffset=+2] -include::modules/error-500-troubleshooting.adoc[leveloffset=+2] -include::modules/error-502-troubleshooting.adoc[leveloffset=+2] -include::modules/build-trigger-error.adoc[leveloffset=+2] -include::modules/build-logs-not-loading.adoc[leveloffset=+2] -include::modules/cannot-access-private-repo.adoc[leveloffset=+2] -include::modules/cannot-locate-dockerfile.adoc[leveloffset=+2] -include::modules/cannot-reach-registry-endpoint.adoc[leveloffset=+2] -include::modules/docker-failing-pulls.adoc[leveloffset=+2] -include::modules/docker-io-timeout.adoc[leveloffset=+2] -include::modules/docker-login-error.adoc[leveloffset=+2] -include::modules/docker-timestamp-error.adoc[leveloffset=+2] -include::modules/marathon-mesos-fail.adoc[leveloffset=+2] -include::modules/mirrored-images-unable-pull-rhocp.adoc[leveloffset=+2] -include::modules/secrets-garbage-collected.adoc[leveloffset=+2] -include::modules/troubleshooting-slow-pushes.adoc[leveloffset=+2] - - - -//how tos -//include::modules/troubleshooting-how-tos.adoc[leveloffset=+2] -//include::modules/how-to-list-quay-repos.adoc[leveloffset=+3] -//include::modules/rotating-log-files.adoc[leveloffset=+3] - -//faqs -include::modules/frequently-asked-questions.adoc[leveloffset=+2] -include::modules/clair-distroless-container-images.adoc[leveloffset=+3] -include::modules/operator-geo-replication.adoc[leveloffset=+3] -include::modules/ldap-timeouts-quay.adoc[leveloffset=+3] -include::modules/limit-organization-creation.adoc[leveloffset=+3] -include::modules/resource-demand-failed-operator.adoc[leveloffset=+3] -include::modules/nested-ldap-team-sync.adoc[leveloffset=+3] -//// \ No newline at end of file +* link:https://access.redhat.com/articles/7018077[Troubleshooting Clair]