Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions specs/crawler/common/parameters.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ CrawlerIdParameter:
schema:
$ref: '#/CrawlerID'

CrawlerLogIdParameter:
name: logId
in: path
description: Crawler log ID.
required: true
schema:
$ref: '#/CrawlerLogID'

TaskIdParameter:
name: taskID
in: path
Expand Down Expand Up @@ -36,6 +44,48 @@ Page:
schema:
$ref: '#/page'

From:
name: from
in: query
description: Date 'from' filter.
schema:
$ref: '#/from'

Until:
name: until
in: query
description: Date 'until' filter.
schema:
$ref: '#/until'

Status:
name: status
in: query
description: Status to filter 'DONE', 'SKIPPED' or 'FAILED'.
schema:
$ref: '#/urlsCrawledGroupStatus'

Limit:
name: limit
in: query
description: Limit of the query results.
schema:
$ref: '#/limit'

Offset:
name: offset
in: query
description: Offset of the query results.
schema:
$ref: '#/offset'

Order:
name: order
in: query
description: Order of the query 'ASC' or 'DESC'.
schema:
$ref: '#/order'

Name:
name: name
in: query
Expand All @@ -60,6 +110,11 @@ CrawlerID:
description: Universally unique identifier (UUID) of the crawler.
example: e0f6db8a-24f5-4092-83a4-1b2c6cb6d809

CrawlerLogID:
type: string
description: Universally unique identifier (UUID) of the crawler log.
example: a2ebb507-ef64-4b6b-9d84-ef66baaa7a80

TaskID:
type: string
description: Universally unique identifier (UUID) of the task.
Expand Down Expand Up @@ -137,6 +192,37 @@ total:
description: Total number of retrievable items.
example: 100

from:
type: string
description: Unix string 'from' date.
example: 1762264044

until:
type: string
description: Unix string 'until' date.
example: 1762264044

limit:
type: integer
description: Limit of the query results.
minimum: 1
default: 10
maximum: 1000
example: 10

offset:
type: integer
description: Offset of the query results.
example: 11

order:
type: string
description: |
Order of the query.
enum:
- ASC
- DESC

Pagination:
type: object
description: Pagination information.
Expand Down
12 changes: 12 additions & 0 deletions specs/crawler/common/schemas/crawlerLogsFileResponse.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
description: Get Crawler File Response.
content:
application/json:
schema:
title: getCrawlerFileResponse
type: object
properties:
file:
title: crawlerLogFile
type: string
required:
- file
69 changes: 69 additions & 0 deletions specs/crawler/common/schemas/crawlerLogsResponse.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
description: List Response of Crawler Logs.
content:
application/json:
schema:
title: listCrawlerLogsResponse
type: object
properties:
logs:
type: array
items:
title: crawlerLogItem
type: object
properties:
id:
type: string
description: ID of the crawler log.
configId:
type: string
description: Crawler Config identifier.
reindexId:
type: string
description: Identifier of Reindex.
fileSizeBytes:
type: integer
description: Size of the compressed crawler log.
uncompressedSizeBytes:
type: integer
description: Size of the uncompressed crawler log.
crawlStartedAt:
type: string
description: Crawl started at date.
crawlCompletedAt:
type: string
description: Crawl started at date.
fileCreatedAt:
type: string
description: File created date.
expiresAt:
type: string
description: File expiration date.
status:
type: string
description: File status.
accessCount:
type: integer
description: File access count.
lastAccessedAt:
type: string
description: File last accessed date.
nullable: true
urlsDone:
type: integer
description: Crawler urls done.
urlsSkipped:
type: integer
description: Crawler urls skipped.
urlsFailed:
type: integer
description: Crawler urls failed.
meta:
title: crawlerLogsMeta
type: object
properties:
total:
type: integer
description: Total of records found.
required:
- logs
- meta
28 changes: 28 additions & 0 deletions specs/crawler/paths/crawlerLogs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
get:
operationId: listCrawlRuns
summary: List crawler runs
description: |
The Crawler Logs feature allows you to monitor and debug your crawler’s activity by recording
detailed logs for each crawl run. Logs are useful for troubleshooting crawl issues,
verifying site coverage, and monitoring crawler performance over time.
tags:
- crawlers
x-acl:
- settings
parameters:
- $ref: '../common/parameters.yml#/CrawlerIdParameter'
- $ref: '../common/parameters.yml#/From'
- $ref: '../common/parameters.yml#/Until'
- $ref: '../common/parameters.yml#/Status'
- $ref: '../common/parameters.yml#/Limit'
- $ref: '../common/parameters.yml#/Offset'
- $ref: '../common/parameters.yml#/Order'
responses:
'200':
$ref: '../common/schemas/crawlerLogsResponse.yml'
'400':
$ref: '../../common/responses/InvalidRequest.yml'
'401':
$ref: '../common/schemas/responses.yml#/MissingAuthorization'
'403':
$ref: '../common/schemas/responses.yml#/NoRightsOnCrawler'
23 changes: 23 additions & 0 deletions specs/crawler/paths/crawlerLogsFile.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
get:
operationId: getCrawlRunFile
summary: Crawler run file
description: |
The Crawler Logs file allows you to monitor and debug your crawler’s activity by recording
detailed logs for each crawl run. Logs are useful for troubleshooting crawl issues,
verifying site coverage, and monitoring crawler performance over time.
tags:
- crawlers
x-acl:
- settings
parameters:
- $ref: '../common/parameters.yml#/CrawlerIdParameter'
- $ref: '../common/parameters.yml#/CrawlerLogIdParameter'
responses:
'200':
$ref: '../common/schemas/crawlerLogsResponse.yml'
'400':
$ref: '../../common/responses/InvalidRequest.yml'
'401':
$ref: '../common/schemas/responses.yml#/MissingAuthorization'
'403':
$ref: '../common/schemas/responses.yml#/NoRightsOnCrawler'
4 changes: 4 additions & 0 deletions specs/crawler/spec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ paths:
$ref: 'paths/crawlerCrawl.yml'
/1/crawlers/{id}/stats/urls:
$ref: 'paths/crawlerStats.yml'
/1/crawlers/{id}/crawl_runs:
$ref: 'paths/crawlerLogs.yml'
/1/crawlers/{id}/{logId}/download:
$ref: 'paths/crawlerLogsFile.yml'
/1/crawlers/{id}/config:
$ref: 'paths/crawlerConfig.yml'
/1/crawlers/{id}/config/versions:
Expand Down