firecrawl-dag

A module to read information from websites using Firecrawl (firecrawl.dev)

Installation

dagger install github.com/kpenfound/dag/firecrawl@v1.0.2

FirecrawlDag !

Name	Type	Default Value	Description
apiKey	Secret !	-	A reference to a secret value, which can be handled more safely than the value itself.

Example

dagger -m github.com/kpenfound/dag/firecrawl@7713920d272b2f990fa50eee5a2da53829168db6 call \
 --api-key env:MYSECRET

func (m *myModule) example(apiKey *Secret) *FirecrawlDag  {
	return dag.
			FirecrawlDag(apiKey)
}

@function
def example(api_key: dagger.Secret) -> dag.FirecrawlDag:
	return (
		dag.firecrawl_dag(api_key)
	)

@func()
example(apiKey: Secret): FirecrawlDag {
	return dag
		.firecrawlDag(apiKey)
}

Crawl an entire website and return the data containing all of its pages.

[FirecrawlDagCrawlPage ! ] !

Name	Type	Default Value	Description
url	String !	-	The URL to crawl.

Example

dagger -m github.com/kpenfound/dag/firecrawl@7713920d272b2f990fa50eee5a2da53829168db6 call \
 --api-key env:MYSECRET crawl --url string

func (m *myModule) example(apiKey *Secret, url string) []*FirecrawlDagCrawlPage  {
	return dag.
			FirecrawlDag(apiKey).
			Crawl(url)
}

@function
def example(api_key: dagger.Secret, url: str) -> List[dag.FirecrawlDagCrawlPage]:
	return (
		dag.firecrawl_dag(api_key)
		.crawl(url)
	)

@func()
example(apiKey: Secret, url: string): FirecrawlDagCrawlPage[] {
	return dag
		.firecrawlDag(apiKey)
		.crawl(url)
}

Map a website to get a list of all the page urls

[String ! ] !

Name	Type	Default Value	Description
url	String !	-	The URL to scrape.

Example

dagger -m github.com/kpenfound/dag/firecrawl@7713920d272b2f990fa50eee5a2da53829168db6 call \
 --api-key env:MYSECRET map --url string

func (m *myModule) example(ctx context.Context, apiKey *Secret, url string) []string  {
	return dag.
			FirecrawlDag(apiKey).
			Map(ctx, url)
}

@function
async def example(api_key: dagger.Secret, url: str) -> List[str]:
	return await (
		dag.firecrawl_dag(api_key)
		.map(url)
	)

@func()
async example(apiKey: Secret, url: string): Promise<string[]> {
	return dag
		.firecrawlDag(apiKey)
		.map(url)
}

Scrape a signle webpage and return the content in markdown format.

String !

Name	Type	Default Value	Description
url	String !	-	The URL to scrape.

Example

dagger -m github.com/kpenfound/dag/firecrawl@7713920d272b2f990fa50eee5a2da53829168db6 call \
 --api-key env:MYSECRET scrape --url string

func (m *myModule) example(ctx context.Context, apiKey *Secret, url string) string  {
	return dag.
			FirecrawlDag(apiKey).
			Scrape(ctx, url)
}

@function
async def example(api_key: dagger.Secret, url: str) -> str:
	return await (
		dag.firecrawl_dag(api_key)
		.scrape(url)
	)

@func()
async example(apiKey: Secret, url: string): Promise<string> {
	return dag
		.firecrawlDag(apiKey)
		.scrape(url)
}