Certtf 756 display number of jobs

server/HACKING.md

@@ -16,7 +16,7 @@ This will setup Testflinger running on port 5000, along with MongoDB and Vault.
 To get this running on your system:
 
 ```shell
-docker-compose up -d --build
+docker compose up -d --build
 ```
 
 If you want to manage client credentials in your local database, you can use

server/devel/create_sample_data.py

@@ -63,7 +63,15 @@ def _server_validator(server: str) -> str:
         "--queues",
         type=int,
         default=10,
-        help="Number of queues to create",
+        help="Number of queues to distribute amongst jobs and agents",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--advertised-queues",
+        type=int,
+        default=1,
+        help="Number of advertised queues to create",
     )
 
     parser.add_argument(
@@ -112,7 +120,9 @@ def __iter__(self):
                 "state": random.choice(("waiting", "test", "provision")),
             }
             if self.queue_list:
-                agent_data["queues"] = [random.choice(self.queue_list)]
+                agent_data["queues"] = random.sample(
+                    self.queue_list, random.randint(1, len(self.queue_list))
+                )
             yield (f"{self.prefix}{agent_num}", agent_data)
 
 
@@ -241,7 +251,10 @@ def main():
     testflinger_client = TestflingerClient(server_url=args.server)
 
     queues = QueueDataGenerator(num_queues=args.queues)
-    testflinger_client.post_queue_data(queues=queues)
+    # configure "advertised" queues:
+    testflinger_client.post_queue_data(
+        random.sample(tuple(queues), random.randint(1, args.advertised_queues))
+    )
     logging.info("Created %s queues", args.queues)
 
     valid_queue_names = extract_queue_names(queues=queues)

server/devel/openapi_app.py

@@ -18,22 +18,24 @@
 
 This module creates a lightweight Testflinger server instance specifically
 for extracting the OpenAPI specification without requiring MongoDB or other
-external dependencies. 
-It's used by the APIFlask to generate the API spec in JSON format. 
+external dependencies.
+It's used by the APIFlask to generate the API spec in JSON format.
 (https://apiflask.com/openapi/#the-flask-spec-command)
 
 Usage:
     uvx --with tox-uv tox -e schema
 """
 
-from dataclasses import dataclass 
+from dataclasses import dataclass
 from testflinger.application import create_flask_app
 
+
 @dataclass
 class OpenAPIConfig:
     """Config for Testing."""
 
     TESTING = True
 
+
 # Create and expose the app in TESTING mode for Flask CLI to use
 app = create_flask_app(OpenAPIConfig)

server/scripts/generate_openapi_schema.py

@@ -53,7 +53,7 @@ def generate_schema() -> dict:
 
 def normalize_json(data: dict) -> str:
     """Normalize JSON to compact form for comparison"""
-    return json.dumps(data, sort_keys=True, separators=(',', ':'))
+    return json.dumps(data, sort_keys=True, separators=(",", ":"))
 
 
 def diff_schemas(local_schema_path: Path) -> bool:
@@ -70,7 +70,10 @@ def diff_schemas(local_schema_path: Path) -> bool:
     generated = generate_schema()
 
     if not local_schema_path.exists():
-        print(f"Error: Expected schema file not found: {local_schema_path}", file=sys.stderr)
+        print(
+            f"Error: Expected schema file not found: {local_schema_path}",
+            file=sys.stderr,
+        )
         return False
 
     with local_schema_path.open() as f:
@@ -83,7 +86,10 @@ def diff_schemas(local_schema_path: Path) -> bool:
     if generated_normalized != local_normalized:
         print("Error: OpenAPI schema is out of date", file=sys.stderr)
         print("", file=sys.stderr)
-        print("To update the schema, run from server/ directory:", file=sys.stderr)
+        print(
+            "To update the schema, run from server/ directory:",
+            file=sys.stderr,
+        )
         print(f"  uvx --with tox-uv tox run -e schema", file=sys.stderr)
         return False
 
@@ -99,13 +105,13 @@ def main():
         "-o",
         type=Path,
         default=None,
-        help="Write schema to specified file (default is stdout)"
+        help="Write schema to specified file (default is stdout)",
     )
     parser.add_argument(
         "--diff",
         "-d",
         type=Path,
-        help="Compare generated schema with the specified file for validation"
+        help="Compare generated schema with the specified file for validation",
     )
 
     args = parser.parse_args()
@@ -124,7 +130,7 @@ def main():
         args.output.parent.mkdir(parents=True, exist_ok=True)
         with args.output.open("w") as f:
             json.dump(schema, f, indent=2, sort_keys=True)
-            f.write("\n")       # trailing linebreak
+            f.write("\n")  # trailing linebreak
         print(f" Schema written to: {args.output}")
     else:
         print(json.dumps(schema, indent=2, sort_keys=True))

server/src/testflinger/database.py

@@ -280,6 +280,18 @@ def get_jobs_on_queue(queue: str) -> list[dict]:
     return list(jobs)
 
 
+def get_num_incomplete_jobs_on_queue(queue: str) -> int:
+    """Get the number of incomplete jobs on a specific queue."""
+    return mongo.db.jobs.count_documents(
+        {
+            "job_data.job_queue": queue,
+            "result_data.job_state": {
+                "$nin": ["complete", "completed", "cancelled"]
+            },
+        }
+    )
+
+
 def calculate_percentiles(data: list) -> dict:
     """
     Calculate the percentiles of the wait times for each queue.

server/src/testflinger/templates/agent_detail.html

@@ -32,16 +32,29 @@ <h1 class="p-heading--3">Agent Detail - {{ agent.name }}</h1>
     </tbody>
   </table>
   <h2 class="p-muted-heading">Queues</h2>
-  <ul>
-    {% for queue in agent.queues %}
-      <li>
-        <a href="{{ url_for('testflinger.queue_detail', queue_name=queue) }}">{{ queue }}</a>
-        {% if agent.restricted_to[queue]|default([]) %}
-          <small>(restricted to: {{ agent.restricted_to[queue]|join(", ") }})</small>
-        {% endif %}
-      </li>
-    {% endfor %}
-  </ul>
+  <table aria-label="Queues table" class="p-table--mobile-card">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Number of jobs</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for queue in agent.queues %}
+        <tr class="searchable-row">
+          <td data-heading="Name">
+            <a href="{{ url_for('testflinger.queue_detail', queue_name=queue.name) }}">{{ queue.name }}</a>
+            {% if agent.restricted_to[queue.name]|default([]) %}
+              <small>(restricted to: {{ agent.restricted_to[queue.name]|join(", ") }})</small>
+            {% endif %}
+          </td>
+          <td data-heading="Number of Unfinished Jobs">{{ queue.numjobs }}</td>
+          <td data-heading="Description">{{ queue.description }}</td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
   <h2 class="p-muted-heading">Provision History</h2>
   <form action="" method="get" class="p-form--inline">
     <div class="p-form__group">
@@ -98,7 +111,7 @@ <h2 class="p-muted-heading">Agent Log</h2>
         <code>
             {{ agent.log|join('\n') }}
         </code>
-    </pre>
+        </pre>
     </div>
   </div>
 {% endblock content %}

server/src/testflinger/views.py

@@ -91,6 +91,20 @@ def agent_detail(agent_id):
         for queue in agent_info.get("queues", [])
     }
 
+    queue_info = []
+    for queue_name in agent_info.pop("queues", []):
+        queue_data = mongo.db.queues.find_one({"name": queue_name})
+        if not queue_data:
+            # If it's not an advertised queue, create some dummy data
+            queue_data = {"description": ""}
+        queue_data["name"] = queue_name
+        queue_data["numjobs"] = database.get_num_incomplete_jobs_on_queue(
+            queue_name
+        )
+        queue_info.append(queue_data)
+
+    agent_info["queues"] = queue_info
+
     # We want to include the start/stop dates so that default values
     # can be filled in for the date pickers
     agent_info["start"] = start_date
@@ -186,13 +200,8 @@ def queues_data():
 
     # Get job counts for each queue
     for queue in queue_data:
-        queue["numjobs"] = mongo.db.jobs.count_documents(
-            {
-                "job_data.job_queue": queue["name"],
-                "result_data.job_state": {
-                    "$nin": ["complete", "completed", "cancelled"]
-                },
-            }
+        queue["numjobs"] = database.get_num_incomplete_jobs_on_queue(
+            queue["name"]
         )
     return queue_data
 

server/tests/test_views.py

@@ -56,16 +56,34 @@ def test_queues():
     )
     mongo.db.jobs.insert_many(
         [
-            {"job_data": {"job_queue": "advertised_queue1"}},
-            {"job_data": {"job_queue": "advertised_queue1"}},
-            {"job_data": {"job_queue": "advertised_queue1"}},
-            {"job_data": {"job_queue": "advertised_queue2"}},
-            {"job_data": {"job_queue": "advertised_queue2"}},
+            {
+                "job_data": {"job_queue": "advertised_queue1"},
+                "result_data": {"job_state": "waiting"},
+            },
+            {
+                "job_data": {"job_queue": "advertised_queue1"},
+                "result_data": {"job_state": "running"},
+            },
+            {
+                "job_data": {"job_queue": "advertised_queue1"},
+                "result_data": {"job_state": "waiting"},
+            },
+            {
+                "job_data": {"job_queue": "advertised_queue2"},
+                "result_data": {"job_state": "running"},
+            },
+            {
+                "job_data": {"job_queue": "advertised_queue2"},
+                "result_data": {"job_state": "waiting"},
+            },
         ]
     )
 
     # Get the data from the function we use to generate the view
-    with patch("testflinger.views.mongo", mongo):
+    with (
+        patch("testflinger.views.mongo", mongo),
+        patch("testflinger.database.mongo", mongo),
+    ):
         data = queues_data()
 
     # Make sure we found all the queues, not just advertised ones
@@ -149,6 +167,51 @@ def test_agent_detail_with_restricted_to(testapp):
     assert "(restricted to: test-client-id)" in html
 
 
+def test_agent_detail_with_non_advertised_queue(testapp):
+    """
+    Test that agent detail page handles both advertised and non-advertised queues.
+    """
+    mongo = mongomock.MongoClient()
+    # Insert one advertised queue
+    mongo.db.queues.insert_one(
+        {"name": "advertised_queue", "description": "advertised description"}
+    )
+    # Agent listens to both advertised and non-advertised queues
+    mongo.db.agents.insert_one(
+        {
+            "name": "agent1",
+            "queues": ["advertised_queue", "non_advertised_queue"],
+            "updated_at": datetime.now(tz=timezone.utc),
+        }
+    )
+    mongo.db.jobs.insert_many(
+        [
+            {
+                "job_data": {"job_queue": "advertised_queue"},
+                "result_data": {"job_state": "waiting"},
+            },
+            {
+                "job_data": {"job_queue": "non_advertised_queue"},
+                "result_data": {"job_state": "running"},
+            },
+        ]
+    )
+
+    with (
+        patch("testflinger.views.mongo", mongo),
+        patch("testflinger.database.mongo", mongo),
+    ):
+        with testapp.test_request_context():
+            response = agent_detail("agent1")
+
+    html = str(response)
+    # Should include both advertised and non-advertised queues
+    assert "advertised_queue" in html
+    assert "non_advertised_queue" in html
+    # Non-advertised queue creates dummy data with empty description
+    assert "advertised description" in html
+
+
 def test_job_not_found(testapp):
     """
     Test that the job_detail fails gracefully when