limit := 1
slog.Debug("Crawling URL", "url", content.URL, "limit", limit, "tx", tx)
crawlResp, err := app.CrawlURL(content.URL, &firecrawl.CrawlParams{
Limit: &limit,
ScrapeOptions: firecrawl.ScrapeParams{
Formats: []string{"markdown"},
},
}, nil)
if err != nil {
slog.Error("failed to crawl URL", "error", err, "url", content.URL, "tx", tx)
continue
}
When trying to scrape the page https://szwichtenberg.de/ph_news/grundfos-hauswasserwerk with
we get a unmarshal error:
json: cannot unmarshal array into Go struct field FirecrawlDocumentMetadata.data.metadata.robots of type string