Revert "Tokenize Detokenize compatibility"

This reverts commit 6068946cd5cc01570606b78e8acde6f07455a6a0.

examples/tokenization/tokenization.ts

@@ -1,21 +0,0 @@
-import ollama from '../../src/browser.js'
-
-async function main() {
-  // Tokenize some text
-  const tokResponse = await ollama.tokenize({
-    model: 'llama3.2',
-    text: 'Hello, how are you?'
-  })
-  
-  console.log('Tokens from model:', tokResponse.tokens)
-
-  // Detokenize the tokens back to text
-  const detokResponse = await ollama.detokenize({
-    model: 'llama3.2', 
-    tokens: tokResponse.tokens
-  })
-
-  console.log('Text from model:', detokResponse.text)
-}
-
-main().catch(console.error)

src/browser.ts

@@ -9,8 +9,6 @@ import type {
   CopyRequest,
   CreateRequest,
   DeleteRequest,
-  DetokenizeRequest,
-  DetokenizeResponse,
   EmbedRequest,
   EmbedResponse,
   EmbeddingsRequest,
@@ -26,8 +24,6 @@ import type {
   ShowRequest,
   ShowResponse,
   StatusResponse,
-  TokenizeRequest,
-  TokenizeResponse,
 } from './interfaces.js'
 
 export class Ollama {
@@ -326,37 +322,9 @@ async encodeImage(image: Uint8Array | string): Promise<string> {
     })
     return (await response.json()) as ListResponse
   }
-
-  /**
-   * Tokenizes text into tokens.
-   * @param request {TokenizeRequest} - The request object.
-   * @returns {Promise<TokenizeResponse>} - The response object.
-   */
-  async tokenize(request: TokenizeRequest): Promise<TokenizeResponse> {
-    const response = await utils.post(this.fetch, `${this.config.host}/api/tokenize`, {
-      ...request,
-    }, {
-      headers: this.config.headers
-    })
-    return (await response.json()) as TokenizeResponse
-  }
-
-  /**
-   * Detokenizes tokens back into text.
-   * @param request {DetokenizeRequest} - The request object.
-   * @returns {Promise<DetokenizeResponse>} - The response object.
-   */
-  async detokenize(request: DetokenizeRequest): Promise<DetokenizeResponse> {
-    const response = await utils.post(this.fetch, `${this.config.host}/api/detokenize`, {
-      ...request,
-    }, {
-      headers: this.config.headers
-    })
-    return (await response.json()) as DetokenizeResponse
-  }
 }
+
 export default new Ollama()
 
 // export all types from the main entry point so that packages importing types dont need to specify paths
 export * from './interfaces.js'
-

src/interfaces.ts

@@ -159,16 +159,6 @@ export interface EmbeddingsRequest {
   options?: Partial<Options>
 }
 
-export interface TokenizeRequest {
-  model: string
-  text: string
-}
-
-export interface DetokenizeRequest {
-  model: string
-  tokens: Int32Array
-}
-
 // response types
 
 export interface GenerateResponse {
@@ -252,14 +242,6 @@ export interface ListResponse {
   models: ModelResponse[]
 }
 
-export interface TokenizeResponse {
-  tokens: Int32Array
-}
-
-export interface DetokenizeResponse {
-  text: string
-}
-
 export interface ErrorResponse {
   error: string
 }