deepcrawl
100% free and full open-source edge Firecrawl alternative with better links extraction for agents - that you can deploy to cloudflare or vercel by yourself.
File Explorer
Download Latest Version (.zip)- config.json
- no-release.md
- README.md
- bug_report.md
- feature_request.md
- deploy-auth-worker.yml
- deploy-v0-worker.yml
- release.yml
- sync-template-main.yml
- validate.yml
- changeset-version.js
- CODE_OF_CONDUCT.md
- ensure-sdk-changeset.mjs
- FUNDING.yml
- SECURITY.md
- pre-commit
- pre-push
- extensions.json
- settings.json
- tasks.json
- page.tsx
- layout.tsx
- account-skeletons.tsx
- multiple-accounts-management-card.tsx
- password-change-card.tsx
- providers-management-card.tsx
- sessions-management-card.tsx
- user-avatar-card.tsx
- user-email-card.tsx
- user-name-card.tsx
- page.tsx
- page.tsx
- page.tsx
- page.tsx
- layout.tsx
- page.tsx
- layout.tsx
- layout.tsx
- page.tsx
- auth.ts
- playground.ts
- route.ts
- route.ts
- route.ts
- route.ts
- route.ts
- route.ts
- favicon.ico
- layout.tsx
- not-found.tsx
- grid-icons.tsx
- motion-grid.tsx
- api-keys-page-client.tsx
- api-keys-table.tsx
- create-api-key-dialog.tsx
- edit-api-key-dialog.tsx
- playground-api-key-bootstrap.tsx
- index.tsx
- forgot-password-form.tsx
- magic-link-form.tsx
- reset-password-form.tsx
- sign-in-form.tsx
- sign-up-form.tsx
- unified-verification-form.tsx
- auth-callback.tsx
- auth-card.tsx
- auth-form.tsx
- back-button.tsx
- last-userd-badge.tsx
- logout.tsx
- magic-link-button.tsx
- passkey-button.tsx
- password-input.tsx
- provider-button.tsx
- social-providers.ts
- tier-labels.tsx
- copy-markdown-button.tsx
- open-docs-menu.tsx
- header-navigation-layout.tsx
- logs-chart.tsx
- logs-columns.tsx
- logs-data-grid-card.tsx
- logs-data-grid-skeleton.tsx
- logs-data-grid.tsx
- logs-date-range-select.tsx
- logs-detail-content.tsx
- logs-filter-popover.tsx
- logs-page-header.tsx
- logs-search-bar.tsx
- logs-toolbar.tsx
- action-buttons.tsx
- content-tabs.tsx
- error-card.tsx
- index.ts
- page-metadata-card.tsx
- pg-response-area.tsx
- task-info-card.tsx
- cache-options-menu.tsx
- cleaning-processor-menu.tsx
- content-format-options-menu.tsx
- detailed-options-accordion.tsx
- detailed-sdk-implementation.tsx
- lazy-playground.tsx
- link-extraction-options-menu.tsx
- markdown-options-menu.tsx
- metrics-number.tsx
- metrics-options-menu.tsx
- operation-selector.tsx
- option-preview-badges.tsx
- playground-operation-client.tsx
- playground-options-menus-toolbar.tsx
- playground-skeleton.tsx
- playground-url-input.tsx
- task-input-options.tsx
- tree-options-menu.tsx
- url-input.tsx
- index.tsx
- query.provider.tsx
- app-sidebar.tsx
- nav-main.tsx
- team-switcher.tsx
- index.tsx
- user-actions.tsx
- user-dropdown.tsx
- create-deepcrawl-cli.tsx
- faq.tsx
- footer.tsx
- hero.tsx
- surfaces.tsx
- toolkit-suite.tsx
- value-prop.tsx
- ascii-pyramid.tsx
- banner.tsx
- border-shader.tsx
- create-deepcrawl-terminal.tsx
- display-cards.tsx
- dotted-map.tsx
- flickering-grid.tsx
- free-tag.tsx
- globe.tsx
- h2.tsx
- header.tsx
- installer.tsx
- lazy-components.tsx
- operation-selector-demo.tsx
- performance-meter.tsx
- sdk-codeblock.tsx
- secure-shield.tsx
- tick.tsx
- tilted-scroll.tsx
- .gitkeep
- copy-button.tsx
- deepcrawl-logo.tsx
- deploy-attribution-banner.tsx
- layout-toggle.tsx
- loading-spinner.tsx
- mdx-components.tsx
- page-elements.tsx
- passkey-cleanup-guide.tsx
- search-trigger.tsx
- spinner-button.tsx
- index.ts
- get-markdown-example.png
- playground-client-example.png
- read-url-example.png
- export-response.mdx
- get-one-log.mdx
- list-logs.mdx
- meta.json
- extract-links.mdx
- get-links.mdx
- meta.json
- get-markdown.mdx
- meta.json
- read-url.mdx
- meta.json
- context-for-ai.mdx
- cron-job.mdx
- meta.json
- nextjs-ai-agents.mdx
- meta.json
- quick-start.mdx
- why-choose-deepcrawl.mdx
- index.mdx
- meta.json
- index.mdx
- meta.json
- architecture.mdx
- meta.json
- index.mdx
- meta.json
- index.mdx
- meta.json
- create-deepcrawl.mdx
- index.mdx
- meta.json
- index.mdx
- meta.json
- meta.json
- index.mdx
- meta.json
- contributing.mdx
- faq.mdx
- meta.json
- index.mdx
- meta.json
- cookies-sessions-auth.mdx
- meta.json
- meta.json
- node-sdk.mdx
- meta.json
- playground.mdx
- index.mdx
- meta.json
- logs-date-range-context.tsx
- logs-filters-context.tsx
- logs-provider.tsx
- playground-context.tsx
- context-validation.tsx
- defaults.ts
- index.ts
- types.ts
- use-deepcrawl-client.ts
- use-execution-timer.ts
- use-extractlinks-options.ts
- use-getmarkdown-options.ts
- use-operation-options.ts
- use-playground-operations.ts
- use-playground-state.ts
- use-readurl-options.ts
- use-task-input-state.ts
- auth.hooks.ts
- logs.hooks.ts
- use-captcha.tsx
- use-header-scroll.ts
- use-hydrated.ts
- use-logs-data.ts
- use-logs-date-range.ts
- use-logs-filters.ts
- use-logs-pagination.ts
- use-logs-sorting.ts
- use-logs-table.ts
- use-success-transition.ts
- config.ts
- types.ts
- operations-config.ts
- scroll-anchors.ts
- auth-errors.ts
- auth-mode.ts
- auth.client-types.ts
- auth.client.ts
- brand.ts
- get-llm-text.ts
- layout.config.tsx
- navigation-config.ts
- passkey-utils.ts
- playground-api-key-policy.ts
- playground-api-key.client.ts
- playground-api-key.ts
- response-samples.ts
- site-config.ts
- source.ts
- file.svg
- globe.svg
- next.svg
- vercel.svg
- window.svg
- auth-query.client.ts
- auth-query.server.ts
- logs-query.client.ts
- logs-query.server.ts
- query-keys.ts
- query-options.client.ts
- query-options.server.ts
- query.client.ts
- auth.ts
- clean.ts
- index.ts
- deep-equal.ts
- error-handler.client.ts
- formatter.ts
- sdk-snippet.ts
- url-input-pre-validation.ts
- clipboard.ts
- index.ts
- use-scroll-to-anchor.ts
- .env.example
- .gitignore
- biome.jsonc
- components.json
- eslint.config.mjs
- next.config.mjs
- package.json
- postcss.config.mjs
- proxy.ts
- README.md
- source.config.ts
- tsconfig.json
- vercel.json
- index.tsx
- create-hono-app.tsx
- context.ts
- auth.ts
- cors.ts
- kv.ts
- loggers.ts
- index.tsx
- rpc.d.ts
- .dev.vars.example
- biome.json
- package.json
- README.md
- tsconfig.json
- worker-configuration.d.ts
- wrangler.jsonc
- generate-openapi-yaml.ts
- worker-config-override.ts
- configure-open-api.ts
- create-hono-app.ts
- generate-openapi-yaml.ts
- types.ts
- create-hono-app.tsx
- configs.ts
- openapi.handler.ts
- orpc.client.ts
- rpc.handler.ts
- context.ts
- api-key-auth.hono.ts
- client.auth.ts
- connInfo.hono.ts
- cookie-auth.hono.ts
- cors.hono.ts
- d1.cloudflare.ts
- error.ts
- jwt-auth.hono.ts
- rate-limit.orpc.ts
- retry.orpc.ts
- service-fetchers.hono.ts
- services.app.ts
- links.handler.ts
- links.processor.ts
- logs.handler.ts
- logs.processor.ts
- read.handler.ts
- read.processor.ts
- index.ts
- activity-logger.service.ts
- base64-image.ts
- link-normalize.ts
- main-content.ts
- tag-filter.ts
- html-cleaning.service.ts
- selectors.ts
- link.service.ts
- markdown-helpers.ts
- metadata.service.ts
- response-record.service.ts
- scrape.service.ts
- global-types.ts
- hash-tools.ts
- response-hash.ts
- links-kv-key.ts
- read-kv-key.ts
- retry.ts
- helpers.ts
- robots-parser.ts
- sitemap-parser.ts
- clean-empty-values.ts
- dynamics-handling.ts
- post-processing.ts
- response-reconstruction.ts
- target-url-helper.ts
- validate-url.ts
- auth-mode.ts
- loggers.ts
- markdown.ts
- metrics.ts
- worker-env.ts
- app.ts
- index.ts
- orpc.ts
- .dev.vars.example
- .npmrc
- biome.json
- openapi_orpc.yaml
- package.json
- README.md
- tsconfig.json
- worker-configuration.d.ts
- wrangler.jsonc
- package.json
- .env.example
- .vars.example
- README.md
- better-auth-url.test.ts
- trusted-origins.test.ts
- auth.client.config.ts
- auth.config.ts
- constants.ts
- index.ts
- providers.config.ts
- auth.ts
- index.ts
- email-verification.tsx
- index.ts
- magic-link.tsx
- organization-invitation.tsx
- password-reset.tsx
- styles.ts
- index.ts
- better-auth-url.ts
- config-validator.ts
- email.ts
- logger.ts
- time-converter.ts
- index.ts
- biome.json
- package.json
- README.md
- tsconfig.json
- index.ts
- utils.ts
- errors.ts
- index.ts
- links.ts
- read.ts
- biome.json
- package.json
- README.md
- tsconfig.json
- create-deepcrawl.js
- optional.md
- prompt-flow.md
- required.md
- README.md
- prepare-template-output.d.ts
- prepare-template-output.mjs
- apply-v0-remote-migrations.test.ts
- brand.test.ts
- cli-args.test.ts
- clone-template.test.ts
- deploy-v0-worker.test.ts
- deployment-success.test.ts
- env-file.test.ts
- exec.test.ts
- jwt-token.test.ts
- messages.test.ts
- patch-v0-wrangler.test.ts
- preflight.test.ts
- prepare-template-output.test.ts
- prepare-v0-local-project.test.ts
- prompt-answers.test.ts
- quick-test-v0-worker.test.ts
- quick-test.test.ts
- select-deployment-target.test.ts
- write-v0-local-jwt-secret.test.ts
- cli-args.ts
- env-file.ts
- exec.ts
- jwt-token.ts
- jwt.ts
- template-source.ts
- terminal-text.ts
- user-abort.ts
- wrangler-config.ts
- apply-v0-remote-migrations.ts
- clone-template.ts
- deploy-v0-worker.ts
- install-deps.ts
- patch-v0-wrangler.ts
- preflight.ts
- prepare-template-output.ts
- prepare-v0-local-project.ts
- provision-v0-resources.ts
- quick-test-v0-worker.ts
- set-wrangler-secret.ts
- write-v0-local-jwt-secret.ts
- brand.ts
- card-format.ts
- confirm-prompt.ts
- deployment-success.ts
- dim.ts
- messages.ts
- prompt-answers.ts
- quick-test.ts
- render-card.ts
- select-deployment-target.ts
- terminal-style.ts
- bin.ts
- run.ts
- package.json
- README.md
- tsconfig.build.json
- tsconfig.json
- 0000_snapshot.json
- 0001_snapshot.json
- 0002_snapshot.json
- 0003_snapshot.json
- 0004_snapshot.json
- 0005_snapshot.json
- 0006_snapshot.json
- 0007_snapshot.json
- _journal.json
- 0000_old_mephisto.sql
- 0001_lowly_stephen_strange.sql
- 0002_plain_reaper.sql
- 0003_loose_blindfold.sql
- 0004_lazy_lifeguard.sql
- 0005_blushing_steel_serpent.sql
- 0006_secret_jamie_braddock.sql
- 0007_keen_kid_colt.sql
- index.ts
- schema.ts
- index.ts
- biome.jsonc
- drizzle.config.ts
- package.json
- README.md
- tsconfig.json
- 0000_snapshot.json
- 0001_snapshot.json
- 0002_snapshot.json
- 0003_snapshot.json
- 0004_snapshot.json
- _journal.json
- 0000_certain_next_avengers.sql
- 0001_gifted_ma_gnuci.sql
- 0002_brown_ultragirl.sql
- 0003_tiny_nightcrawler.sql
- 0004_right_colleen_wing.sql
- index.ts
- schema.ts
- index.ts
- biome.jsonc
- drizzle.config.ts
- package.json
- README.md
- SCHEMA.md
- tsconfig.json
- package.json
- base.js
- next.js
- package.json
- react-internal.js
- README.md
- api-rate-limit.ts
- auth-mode.ts
- auth-rate-limit.ts
- brand.ts
- env.ts
- index.ts
- urls.ts
- vars.ts
- package.json
- README.md
- tsconfig.json
- deepcrawl.test.ts
- integration.test.ts
- setup.ts
- types.test.ts
- index.ts
- deepcrawl.ts
- index.ts
- schemas-public.ts
- types-public.ts
- types-utils.ts
- zod-v4.ts
- biome.json
- CHANGELOG.md
- package.json
- README.md
- tsconfig.json
- tsdown.config.ts
- vitest.config.ts
- package.json
- index.ts
- response-schemas.ts
- types.ts
- utils.ts
- _default_backup.ts
- default.ts
- index.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- utils.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- schemas.ts
- types.ts
- index.ts
- index.ts
- schemas.ts
- types.ts
- biome.json
- package.json
- README.md
- tsconfig.json
- base.json
- nextjs.json
- package.json
- react-library.json
- README.md
- code-block.tsx
- prompt-input.tsx
- response.tsx
- focus-tabs.tsx
- icon-hover-button.tsx
- chart-line.tsx
- clock.tsx
- cpu.tsx
- file-pen-line.tsx
- file-text.tsx
- github-icon.tsx
- google-icon.tsx
- layout-panel-left.tsx
- layout-panel-top.tsx
- link.tsx
- list-tree-icon.tsx
- markdown.tsx
- provider-icons.tsx
- settings.tsx
- endpoint.tsx
- card.tsx
- data-grid-column-filter.tsx
- data-grid-column-header.tsx
- data-grid-column-visibility.tsx
- data-grid-pagination.tsx
- data-grid-table-dnd-rows.tsx
- data-grid-table-dnd.tsx
- data-grid-table.tsx
- data-grid.tsx
- tailwind-indicator.tsx
- toggle.tsx
- .gitkeep
- accordion.tsx
- alert-dialog.tsx
- alert.tsx
- avatar.tsx
- badge.tsx
- breadcrumb.tsx
- button.tsx
- calendar.tsx
- card.tsx
- carousel.tsx
- chart.tsx
- checkbox.tsx
- collapsible.tsx
- command.tsx
- dialog.tsx
- drawer.tsx
- dropdown-menu.tsx
- empty.tsx
- form.tsx
- input-group.tsx
- input.tsx
- kbd.tsx
- label.tsx
- popover.tsx
- responsive-dialog.tsx
- scroll-area.tsx
- select.tsx
- separator.tsx
- sheet.tsx
- sidebar.tsx
- skeleton.tsx
- sonner.tsx
- switch.tsx
- table.tsx
- tabs.tsx
- textarea.tsx
- toggle-group.tsx
- toggle.tsx
- tooltip.tsx
- tree-view.tsx
- .gitkeep
- use-inset-resize.ts
- use-is-mac.ts
- use-media-query.ts
- use-mobile.ts
- context.ts
- utils.ts
- globals.css
- biome.jsonc
- components.json
- eslint.config.js
- package.json
- postcss.config.mjs
- README.md
- tsconfig.json
- tsconfig.lint.json
- og.jpg
- runtime-env.test.ts
- wrangler-jsonc-services.test.ts
- wrangler-jsonc-vars.test.ts
- _dotenv.ts
- _jsonc.ts
- _overrides.ts
- _vars.ts
- _wrangler-jsonc-services.ts
- _wrangler-jsonc-vars.ts
- bootstrap-env-vars.ts
- generate-env-examples.ts
- sync-cloudflare-secrets.ts
- sync-local-env.ts
- sync-to-vars.ts
- sync-vercel-env.ts
- clean.ts
- jwt-mint.ts
- .gitignore
- .npmrc
- AGENTS.md
- AUTH_MODES_AND_WORKERS.md
- biome.jsonc
- CLAUDE.md
- CONTRIBUTING.md
- deepcrawl.code-workspace
- LICENSE
- lint-staged.config.cjs
- package.json
- pnpm-lock.yaml
- pnpm-workspace.yaml
- README.md
- tsconfig.json
- turbo.json
# Installation Guide
1. Get the code
git clone https://github.com/lumpinif/deepcrawl
Downloads the entire project code from GitHub to your computer.
cd deepcrawl
Moves into the project folder you just downloaded.
2. Node.js
Easy RecommendedPrerequisites
cd apps/app
This project's files live in a subfolder, so move into it first.
npm install
Downloads and installs the libraries listed in package.json.
npm start
Starts the development/run server.
After running the command, open the address shown in the terminal (usually something like http://localhost:3000) in your browser.
// repository documentation
Was this content helpful?
(0 ratings)