node-llama-cpp
Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level
File Explorer
Download Latest Version (.zip)- typedoc.css
- typedoc.json
- bug-report.yml
- config.yml
- documentation-issue.yml
- feature-request.yml
- build.yml
- prLint.yml
- test.yml
- FUNDING.yml
- PULL_REQUEST_TEMPLATE.md
- commit-msg
- ogTemplate.svg
- ogTemplate.v1.svg
- ogTemplate.v2.svg
- social.poster.svg
- BlogEntry.vue
- CommentsSection.vue
- DataBadge.vue
- getElectronExampleAppDownloadLink.ts
- HomePage.vue
- LatestVersionHomeBadge.vue
- YouTubePlayer.vue
- apiLinksTransformer.ts
- apiReferenceSidebar.ts
- BlogPageInfoPlugin.ts
- createCustomTwoSlashRenderer.ts
- getBlogPosts.ts
- sidebar.ts
- theme-pattern.dark.svg
- theme-pattern.light.svg
- index.ts
- LayoutContainer.vue
- smoothLoad.css
- style.css
- buildHtmlHeading.ts
- buildHtmlTable.ts
- ensureLocalImage.ts
- getCommandHtmlDoc.ts
- getExcerptFromMarkdownFile.ts
- getInlineCodeBlockHtml.ts
- getMarkdownRenderer.ts
- htmlEscape.ts
- htmlEscapeWithCodeMarkdown.ts
- parseCmakeListsTxtOptions.ts
- renderHtmlTag.ts
- components.d.ts
- config.ts
- tsconfig.json
- electronTemplate.icon.png
- icon.svg
- icon.v3.svg
- logo.png
- logo.roundEdges.png
- logo.v3.png
- logo.v3.roundEdges.avif
- logo.v3.roundEdges.png
- social.poster.png
- star.please.png
- star.please.roundEdges.png
- blog.data.ts
- index.md
- v3.12-gpt-oss.md
- v3.19-gemma-4.md
- v3.6-deepseek-r1.md
- v3.md
- estimate.md
- gguf.md
- gpu.md
- measure.md
- build.md
- clear.md
- download.md
- chat.md
- cli.data.ts
- complete.md
- index.md
- infill.md
- init.md
- inspect.md
- pull.md
- source.md
- awesome.md
- batching.md
- building-from-source.md
- chat-context-shift.md
- chat-session.md
- chat-wrapper.md
- choosing-a-model.md
- cmakeOptions.data.ts
- contributing.md
- CUDA.md
- development.md
- docker.md
- downloading-models.md
- electron.md
- embedding.md
- external-chat-state.md
- function-calling.md
- grammar.md
- index.md
- llama-text.md
- low-level-api.md
- Metal.md
- objects-lifecycle.md
- text-completion.md
- tips-and-tricks.md
- token-bias.md
- token-prediction.md
- tokens.md
- troubleshooting.md
- Vulkan.md
- dark.css
- light.css
- dark.css
- light.css
- style.css
- favicon.ico
- favicon.png
- favicon.svg
- icon.svg
- logo.jpg
- robots.txt
- index.md
- addonEnv.cpp
- addonEnv.h
- addonLog.cpp
- addonLog.h
- addonProgress.cpp
- addonProgress.h
- getGpuInfo.cpp
- getGpuInfo.h
- getProcessMemoryInfo.cpp
- getProcessMemoryInfo.h
- getSwapInfo.cpp
- getSwapInfo.h
- getSystemMemoryInfo.cpp
- getSystemMemoryInfo.h
- addon.cpp
- AddonContext.cpp
- AddonContext.h
- AddonGgufMetadata.cpp
- AddonGgufMetadata.h
- addonGlobals.cpp
- addonGlobals.h
- AddonGrammar.cpp
- AddonGrammar.h
- AddonGrammarEvaluationState.cpp
- AddonGrammarEvaluationState.h
- AddonModel.cpp
- AddonModel.h
- AddonModelData.cpp
- AddonModelData.h
- AddonModelLora.cpp
- AddonModelLora.h
- AddonSampler.cpp
- AddonSampler.h
- RingBuffer.h
- addVariantSuffix.cmake
- win32.ensureNinjaPath.cmake
- win32.ensureNodeLib.cmake
- win32.llvmApplyGnuModeAdaptations.cmake
- win32.llvmEnsureCmakeAr.cmake
- win32.llvmUseGnuModeCompilers.cmake
- win32.programFilesPaths.cmake
- vulkan-gpu-info.cpp
- vulkan-gpu-info.h
- PR-22566.diff
- llvm.win32.host-arm64.target-arm64.cmake
- llvm.win32.host-x64.target-arm64.cmake
- llvm.win32.host-x64.target-x64.cmake
- darwin.host-x64.target-arm64.cmake
- linux.host-arm64.target-arm7l.cmake
- linux.host-arm64.target-x64.cmake
- linux.host-x64.target-arm64.cmake
- linux.host-x64.target-arm7l.cmake
- llvm.win32.host-x64.target-x64.cmake
- win32.host-arm64.target-arm64.cmake
- win32.host-x64.target-arm64.cmake
- package.json
- .clang-format
- binariesGithubRelease.json
- CMakeLists.txt
- package.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- cli.ts
- index.ts
- .gitignore
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- @semantic-release+github+11.0.0.patch
- @semantic-release+npm+13.1.2.patch
- semantic-release+24.1.1.patch
- movePrebuiltBinariesToStandaloneModules.ts
- packTemplates.ts
- postVersion.ts
- prepareCreateNodeLlamaCppModuleForPublish.ts
- prepareStandalonePrebuiltBinaryModules.ts
- publishStandalonePrebuiltBinaryModules.ts
- rerunSemanticReleaseGithubSuccess.ts
- resolveLatestReleaseVersion.ts
- resolveNextReleaseVersion.ts
- scaffoldElectronExampleForCiBuild.ts
- applyLlamaCppRepoPatches.ts
- asyncEvery.ts
- asyncSome.ts
- binariesGithubRelease.ts
- clearAllLocalBuilds.ts
- cloneLlamaCppRepo.ts
- compileLLamaCpp.ts
- detectAvailableComputeLayers.ts
- detectBuildTools.ts
- detectGlibc.ts
- disposeBeforeExit.ts
- getBestComputeLayersAvailable.ts
- getBuildFolderNameForBuildOptions.ts
- getCanUsePrebuiltBinaries.ts
- getExampleUsageCodeOfGetLlama.ts
- getGpuTypesToUseForOption.ts
- getLinuxDistroInfo.ts
- getLlamaGpuTypes.ts
- getLlamaWithoutBackend.ts
- getPlatform.ts
- getPlatformInfo.ts
- hasBuildingFromSourceDependenciesInstalled.ts
- hasFileInPath.ts
- lastBuildInfo.ts
- logBinaryUsageExampleToConsole.ts
- logDistroInstallInstruction.ts
- MemoryOrchestrator.ts
- NoBinaryFoundError.ts
- resolveActualBindingBinaryPath.ts
- resolveCustomCmakeOptions.ts
- testBindingBinary.ts
- testCmakeBinary.ts
- AddonTypes.ts
- consts.ts
- getLlama.ts
- Llama.ts
- types.ts
- chatHistoryFunctionCallMessageTemplate.ts
- extractFunctionCallSettingsFromJinjaTemplate.ts
- extractSegmentSettingsFromTokenizerAndChatTemplate.ts
- getFirstValidResult.ts
- squashChatHistoryItems.ts
- templateSegmentOptionsToChatWrapperSettings.ts
- UniqueIdGenerator.ts
- JinjaTemplateChatWrapper.ts
- TemplateChatWrapper.ts
- ChatModelFunctionsDocumentationGenerator.ts
- chunkChatItems.ts
- getModelLinageNames.ts
- isJinjaTemplateEquivalentToSpecializedChatWrapper.ts
- isLlama3_2LightweightModel.ts
- jsonDumps.ts
- replaceRegularTextInLlamaText.ts
- resolveChatWrapper.ts
- AlpacaChatWrapper.ts
- ChatMLChatWrapper.ts
- DeepSeekChatWrapper.ts
- EmptyChatWrapper.ts
- FalconChatWrapper.ts
- FunctionaryChatWrapper.ts
- Gemma4ChatWrapper.ts
- GemmaChatWrapper.ts
- GeneralChatWrapper.ts
- HarmonyChatWrapper.ts
- Llama2ChatWrapper.ts
- Llama3_1ChatWrapper.ts
- Llama3_2LightweightChatWrapper.ts
- Llama3ChatWrapper.ts
- MistralChatWrapper.ts
- MuseChatWrapper.ts
- QwenChatWrapper.ts
- SeedChatWrapper.ts
- InspectEstimateCommand.ts
- InspectGgufCommand.ts
- InspectGpuCommand.ts
- InspectMeasureCommand.ts
- InspectCommand.ts
- BuildCommand.ts
- ClearCommand.ts
- DownloadCommand.ts
- SourceCommand.ts
- ChatCommand.ts
- CompleteCommand.ts
- DebugCommand.ts
- InfillCommand.ts
- InitCommand.ts
- OnPostInstallCommand.ts
- PullCommand.ts
- basicChooseFromListConsoleInteraction.ts
- ConsoleInteraction.ts
- consolePromptQuestion.ts
- ConsoleTable.ts
- getReadablePath.ts
- interactivelyAskForModel.ts
- isRunningUnderRosetta.ts
- logUsedGpuTypeOption.ts
- packageJsonConfig.ts
- packageManager.ts
- parseXtcArg.ts
- printCommonInfoLines.ts
- printInfoLine.ts
- printModelDestination.ts
- projectTemplates.ts
- renderModelCompatibilityPercentageWithColors.ts
- resolveCommandGgufPath.ts
- resolveHeaderFlag.ts
- resolveModelRecommendationFileOptions.ts
- resolveNpmrcConfig.ts
- splitAnsiToLines.ts
- toBytes.ts
- withCliCommandDescriptionDocsUrl.ts
- cli.ts
- projectTemplates.ts
- recommendedModels.ts
- startCreateCli.ts
- eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy.ts
- FunctionCallNameGrammar.ts
- FunctionCallParamsGrammar.ts
- LlamaFunctionCallValidationError.ts
- LlamaChat.ts
- defineChatSessionFunction.ts
- LlamaChatSessionPromptCompletionEngine.ts
- LlamaChatSession.ts
- DraftSequenceTokenPredictor.ts
- InputLookupTokenPredictor.ts
- firstInFirstOutStrategy.ts
- maximumParallelismStrategy.ts
- padSafeContextSize.ts
- resolveBatchItemsPrioritizationStrategy.ts
- LlamaContext.ts
- LlamaContextSequenceCheckpoints.ts
- LlamaSampler.ts
- TokenPredictor.ts
- types.ts
- TokenAttributes.ts
- LlamaModel.ts
- chunkDocument.ts
- LlamaCompletion.ts
- LlamaEmbedding.ts
- LlamaEmbeddingContext.ts
- LlamaGrammar.ts
- LlamaGrammarEvaluationState.ts
- LlamaJsonSchemaGrammar.ts
- LlamaRankingContext.ts
- TokenBias.ts
- TokenMeter.ts
- InvalidGgufMagicError.ts
- UnsupportedGgufValueTypeError.ts
- GgufFileReader.ts
- GgufFsFileReader.ts
- GgufNetworkFetchFileReader.ts
- getRamUsageFromUnifiedVram.ts
- resolveContextContextSizeOption.ts
- resolveModelGpuLayersOption.ts
- scoreLevels.ts
- GgufInsights.ts
- GgufInsightsConfigurationResolver.ts
- GgufInsightsTokens.ts
- GgufV2Parser.ts
- GgufV3Parser.ts
- parseGguf.ts
- GgufFileInfoTypes.ts
- GgufMetadataTypes.ts
- GgufTensorInfoTypes.ts
- convertMetadataKeyValueRecordToNestedObject.ts
- getGgufFileTypeName.ts
- getGgufMetadataArchitectureData.ts
- getGgufMetadataKeyValue.ts
- ggufQuantNames.ts
- GgufReadOffset.ts
- normalizeGgufDownloadUrl.ts
- resolveBinarySplitGgufPartUrls.ts
- resolveSplitGgufParts.ts
- consts.ts
- readGgufFileInfo.ts
- GbnfAnyJson.ts
- GbnfArray.ts
- GbnfBoolean.ts
- GbnfBooleanValue.ts
- GbnfCommaWhitespace.ts
- gbnfConsts.ts
- GbnfFormatString.ts
- GbnfGrammar.ts
- GbnfInsideStringChar.ts
- GbnfNull.ts
- GbnfNumber.ts
- GbnfNumberValue.ts
- GbnfObjectMap.ts
- GbnfOr.ts
- GbnfRef.ts
- GbnfRepetition.ts
- GbnfString.ts
- GbnfStringValue.ts
- GbnfVerbatimText.ts
- GbnfWhitespace.ts
- defsScope.ts
- GbnfJsonScopeState.ts
- getGbnfJsonTerminalForGbnfJsonSchema.ts
- getGbnfJsonTerminalForLiteral.ts
- validateObjectAgainstGbnfSchema.ts
- GbnfGrammarGenerator.ts
- GbnfTerminal.ts
- getGbnfGrammarForGbnfJsonSchema.ts
- types.ts
- appendUserMessageToChatHistory.ts
- clearTempFolder.ts
- cmake.ts
- compareTokens.ts
- createModelDownloader.ts
- DisposeGuard.ts
- findBestOption.ts
- findCharacterRemovalCountToFitChatHistoryInContext.ts
- formatModelParameterCount.ts
- getBuildDefaults.ts
- getConsoleLogPrefix.ts
- getFirstWritableDir.ts
- getGrammarsFolder.ts
- getLlamaClasses.ts
- getModuleVersion.ts
- getQueuedTokensBeforeStopTrigger.ts
- getReadableContextSize.ts
- getStandardizedChatWrapperSegmentDefinition.ts
- getTempDir.ts
- getTypeScriptTypeStringForGbnfJsonSchema.ts
- GitHubClient.ts
- gitReleaseBundles.ts
- hashString.ts
- includesText.ts
- InsufficientMemoryError.ts
- isLockfileActive.ts
- isToken.ts
- isUrl.ts
- LlamaText.ts
- LruCache.ts
- mergeUnionTypes.ts
- modelDownloadEndpoints.ts
- modelFileAccessTokens.ts
- OpenAIFormat.ts
- optionsMatrix.ts
- OverridesObject.ts
- parseModelFileName.ts
- parseModelUri.ts
- parseTextTemplate.ts
- prettyPrintObject.ts
- ProgressTracker.ts
- pushAll.ts
- removeNullFields.ts
- ReplHistory.ts
- resolveGithubRelease.ts
- resolveLastTokens.ts
- resolveModelDestination.ts
- resolveModelFile.ts
- runtime.ts
- safeEventCallback.ts
- signalSleep.ts
- spawnCommand.ts
- StopGenerationDetector.ts
- ThreadsSplitter.ts
- tokenizeInput.ts
- tokenizerUtils.ts
- TokenStreamRegulator.ts
- transformPromisable.ts
- truncateTextAndRoundToWords.ts
- UnsupportedError.ts
- utilTypes.ts
- waitForLockfileRelease.ts
- withAbortSignal.ts
- withLockfile.ts
- withOra.ts
- withProgressLog.ts
- withStatusLogs.ts
- wrapAbortSignal.ts
- apiDocsIndex.ts
- ChatWrapper.ts
- commands.ts
- config.ts
- consts.ts
- index.ts
- state.ts
- types.ts
- modelFunctions.ts
- llmRpc.ts
- llmState.ts
- createElectronSideBirpc.ts
- electron-env.d.ts
- index.ts
- preload.ts
- vite.svg
- ModelMessageCopyButton.css
- ModelMessageCopyButton.tsx
- ModelMessage.css
- ModelMessage.tsx
- ModelResponseComment.css
- ModelResponseComment.tsx
- ModelResponseThought.css
- ModelResponseThought.tsx
- UserMessage.css
- UserMessage.tsx
- ChatHistory.css
- ChatHistory.tsx
- FixedDivWithSpacer.tsx
- UpdateBadge.css
- UpdateBadge.tsx
- Header.css
- Header.tsx
- InputRow.css
- InputRow.tsx
- MarkdownContent.css
- MarkdownContent.tsx
- MessageMarkdown.css
- MessageMarkdown.tsx
- App.css
- App.tsx
- useExternalState.ts
- AbortIconSVG.tsx
- AddMessageIconSVG.tsx
- CheckIconSVG.tsx
- CopyIconSVG.tsx
- DeleteIconSVG.tsx
- DownloadIconSVG.tsx
- LoadFileIconSVG.tsx
- RightChevronIconSVG.tsx
- SearchIconSVG.tsx
- StarIconSVG.tsx
- llmRpc.ts
- llmState.ts
- createRendererSideBirpc.ts
- index.css
- index.html
- index.tsx
- vite-env.d.ts
- .editorconfig
- .gitignore
- electron-builder.ts
- eslint.config.js
- package.json
- README.md
- tsconfig.json
- tsconfig.node.json
- vite.config.ts
- index.ts
- .editorconfig
- .gitignore
- eslint.config.js
- package.json
- README.md
- tsconfig.json
- .gitignore
- package-lock.json
- package.json
- README.md
- embedding.test.ts
- rank.test.ts
- completion.test.ts
- parallel.test.ts
- ggufParser.test.ts.snap
- ggufInsights.test.ts
- ggufParser.test.ts
- chatSession.test.ts
- embedding.test.ts
- functionaryModelGpuLayersOptions.test.ts
- functions.test.ts
- grammar.test.ts
- sanity.test.ts
- functions.test.ts
- chatSession.test.ts
- functions.test.ts
- grammar.test.ts
- lora.test.ts
- chunkDocument.test.ts
- completion.test.ts
- controlledEvaluate.test.ts
- evaluateWithMetadata.test.ts
- tokenBias.test.ts
- tokenPredictor.test.ts
- completion.test.ts
- promptCompletion.test.ts
- sequenceState.test.ts
- embedding.test.ts
- functions.test.ts
- reasoningBudget.test.ts
- checkpoints.test.ts
- asyncContextLoad.test.ts
- asyncModelLoad.test.ts
- completion.test.ts
- metadataOverrides.test.ts
- stableCodeModelGpuLayersOptions.test.ts
- extractFunctionCallSettingsFromJinjaTemplate.test.ts
- extractSegmentSettingsFromTokenizerAndChatTemplate.test.ts
- getChatWrapperSegmentsOptionsFromTemplateOption.test.ts
- JinjaTemplateChatWrapper.test.ts
- TemplateChatWrapper.test.ts
- jinjaTemplates.ts
- resolveChatWrapper.test.ts
- ChatMLChatWrapper.test.ts
- DeepSeekChatWrapper.test.ts
- FalconChatWrapper.test.ts
- FunctionaryChatWrapper.test.ts
- Gemma4ChatWrapper.test.ts
- GemmaChatWrapper.test.ts
- GeneralChatWrapper.test.ts
- HarmonyChatWrapper.test.ts
- Llama2ChatWrapper.test.ts
- Llama3_1ChatWrapper.test.ts
- Llama3ChatWrapper.test.ts
- MistralChatWrapper.test.ts
- MuseChatWrapper.test.ts
- QwenChatWrapper.test.ts
- recommendedModels.test.ts
- ggufStandaloneParser.test.ts.snap
- ggufStandaloneParser.test.ts
- functionCallGrammar.test.ts
- LlamaGrammar.test.ts
- createSplitPartFilename.test.ts
- LlamaText.test.ts
- modelDownloader.test.ts
- optionsMatrix.test.ts
- parseModelUri.test.ts
- parseTextTemplate.test.ts
- ThreadsSplitter.test.ts
- parseModelFileName.test.ts
- getTempTestDir.ts
- llamaTextSerializer.ts
- simplifyGgufInfoForTestSnapshot.ts
- SpecialTokenSerializer.ts
- SpecialTokensTextSerializer.ts
- testSetup.ts
- message.json
- longSystemPrompt.txt
- downloadAllTestModels.ts
- getTestLlama.ts
- modelFiles.ts
- setupAndTestOnPaperspace.sh
- .commitlintrc.json
- .editorconfig
- .gitignore
- .prettierignore
- .releaserc.ts
- CONTRIBUTING.md
- eslint.config.js
- giscus.json
- LICENSE
- package-lock.json
- package.json
- README.md
- tsconfig.json
- vitest.config.ts
# Installation Guide
1. Get the code
git clone https://github.com/withcatai/node-llama-cpp
Downloads the entire project code from GitHub to your computer.
cd node-llama-cpp
Moves into the project folder you just downloaded.
2. CMake
Medium RecommendedPrerequisites
<sub>Pre-built bindings are provided with a fallback to building from source with cmake</sub>
Type this command into your terminal and run it.
Check that an executable was created inside the build folder, then run it directly (e.g. ./build/app_name).
Pulled directly from this repo's README.
3. Node.js
EasyPrerequisites
npm install node-llama-cpp
Downloads and installs the libraries listed in package.json.
After running the command, open the address shown in the terminal (usually something like http://localhost:3000) in your browser.
Pulled directly from this repo's README.
// repository documentation
Was this content helpful?
(0 ratings)