Llama 2 - Developer Guide

ssh -i <your key name> ubuntu@<Public IP address>

sudo /root/certificate_generate_standalone.sh

{
  "prompt": "\n\n### Instructions:\nWhat is the capital of France?\n\n### Response:\n",
  "stop": ["\n", "###"]
}

{
    "id": "cmpl-91cb1316-fff4-450d-8c01-ad98737afb9d",
    "object": "text_completion",
    "created": 1697623333,
    "model": "/root/models/llama-2-13b-chat.Q5_K_M.gguf",
    "choices": [
        {
            "text": "The capital of France is Paris.",
            "index": 0,
            "logprobs": null,
            "finish_reason": "stop"
        }
    ],
    "usage": {
        "prompt_tokens": 25,
        "completion_tokens": 7,
        "total_tokens": 32
    }
}

{
  "prompt": "\n\n### Instructions:\nWhat is the capital of France?\n\n### Response:\n",
  "stop": ["\n", "###"]
}

{
    "id": "cmpl-91cb1316-fff4-450d-8c01-ad98737afb9d",
    "object": "text_completion",
    "created": 1697623333,
    "model": "/root/models/llama-2-13b-chat.Q5_K_M.gguf",
    "choices": [
        {
            "text": "The capital of France is Paris.",
            "index": 0,
            "logprobs": null,
            "finish_reason": "stop"
        }
    ],
    "usage": {
        "prompt_tokens": 25,
        "completion_tokens": 7,
        "total_tokens": 32
    }
}

{
  "input": "The food was delicious and the waiter..."
}

{
    "object": "list",
    "data": [
        {
            "object": "embedding",
            "embedding": [
                -0.07521496713161469,
                0.44098934531211853,
                0.6786724328994751
            ],
            "index": 0
        }
    ],
    "model": "/root/models/llama-2-13b-chat.Q5_K_M.gguf",
    "usage": {
        "prompt_tokens": 11,
        "total_tokens": 11
    }
}

{
  "messages": [
    {
      "content": "You are a helpful assistant.",
      "role": "system"
    },
    {
      "content": "What is the capital of France?",
      "role": "user"
    }
  ]
}

{
    "id": "chatcmpl-68a41146-ddc6-43e4-b12e-760be6e59e57",
    "object": "chat.completion",
    "created": 1697625763,
    "model": "/root/models/llama-2-13b-chat.Q5_K_M.gguf",
    "choices": [
        {
            "index": 0,
            "message": {
                "role": "assistant",
                "content": "Bonjour! The capital of France is Paris (pronounced \"pah-ree\"). It's a beautiful city known for its art, fashion, cuisine, and rich history. Do you have any other questions about France or Paris?"
            },
            "finish_reason": "stop"
        }
    ],
    "usage": {
        "prompt_tokens": 37,
        "completion_tokens": 85,
        "total_tokens": 122
    }
}

{
    "object": "list",
    "data": [
        {
            "id": "/root/models/llama-2-13b-chat.Q5_K_M.gguf",
            "object": "model",
            "owned_by": "me",
            "permissions": []
        }
    ]
}

npm install

npm start

const axios = require('axios');
require('dotenv').config();

const makePostRequest = async (url, data, timeout) => {
  try {
    const response = await axios.post(url, data, { timeout });
    return { success: response.status === 200, data: response.data };
  } catch (error) {
    return { success: false, error: error.message };
  }
};

const makeGetRequest = async (url, timeout) => {
  try {
    const response = await axios.get(url, { timeout });
    return { success: response.status === 200, data: response.data };
  } catch (error) {
    return { success: false, error: error.message };
  }
};

const printResponseData = (endpoint, data) => {
  console.log(`Response for ${endpoint}:`);
  console.log(JSON.stringify(data, null, 2));
  console.log('');
};

const checkEndpoints = async () => {
  const baseUrl = process.env.BASE_URL;
  const model = process.env.MODEL;

  const endpoints = [
    { path: '/completions', method: makePostRequest, data: { "model": model, "prompt": process.env.PROMPT1 }, printEnv: 'PRINT_COMPLETIONS_RESPONSE' },
    { path: '/embeddings', method: makePostRequest, data: { "input": process.env.PROMPT2, "model": model }, printEnv: 'PRINT_EMBEDDINGS_RESPONSE' },
    { path: '/chat/completions', method: makePostRequest, data: { "messages": [{ "content": "You are a helpful assistant.", "role": "system" }, { "content": process.env.PROMPT1, "role": "user" }], "model": model }, printEnv: 'PRINT_CHAT_COMPLETIONS_RESPONSE' },
    { path: '/models', method: makeGetRequest, printEnv: 'PRINT_MODELS_RESPONSE' }
  ];

  for (const endpoint of endpoints) {
    const url = `${baseUrl}${endpoint.path}`;
    const { success, data, error } = await endpoint.method(url, endpoint.method === makePostRequest ? endpoint.data : null, process.env.REQUEST_TIMEOUT || 50000);
    const printResponse = process.env[endpoint.printEnv] === 'true';

    if (success) {
      console.log(`*** Endpoint ${endpoint.path} is reachable.`);
      if (printResponse) {
        printResponseData(endpoint.path, data);
      }
      console.log('');
    } else {
      console.log(`*** Endpoint ${endpoint.path} is not reachable. Error:`, error);
    }
  }
};

checkEndpoints();

{
  "name": "test-llama",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "start": "node app.js",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "axios": "^1.6.7",
    "dotenv": "^16.4.1"
  }
}

# Base URL for the API
BASE_URL=https://mixtral-test-prod.meetrix.io/v1

# Model to be used in requests
MODEL=mixtral-8x7b-instruct-v0.1

# Prompts for different endpoints
# /completions and /chat/completions
PROMPT1=What is the capital of France?
# /embeddings
PROMPT2=The food was delicious and the waiter...

# Whether to print responses for each endpoint
PRINT_COMPLETIONS_RESPONSE=true
PRINT_EMBEDDINGS_RESPONSE=false
PRINT_CHAT_COMPLETIONS_RESPONSE=true
PRINT_MODELS_RESPONSE=true

# Timeout for requests in milliseconds (default is 50000)
REQUEST_TIMEOUT=50000

ssh -i <your key name> ubuntu@<Public IP address>

sudo tail -f /var/log/syslog