73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
|
# utils/image_handling.py
|
||
|
|
|
||
|
|
import aiohttp
|
||
|
|
import base64
|
||
|
|
|
||
|
|
import globals
|
||
|
|
from utils.core import switch_model
|
||
|
|
from utils.core import miku_vectorstore
|
||
|
|
|
||
|
|
|
||
|
|
async def download_and_encode_image(url):
|
||
|
|
async with aiohttp.ClientSession() as session:
|
||
|
|
async with session.get(url) as resp:
|
||
|
|
if resp.status != 200:
|
||
|
|
return None
|
||
|
|
img_bytes = await resp.read()
|
||
|
|
return base64.b64encode(img_bytes).decode('utf-8')
|
||
|
|
|
||
|
|
async def analyze_image_with_qwen(base64_img):
|
||
|
|
await switch_model("moondream")
|
||
|
|
|
||
|
|
payload = {
|
||
|
|
"model": "moondream",
|
||
|
|
"prompt": "Describe this image in detail.",
|
||
|
|
"images": [base64_img],
|
||
|
|
"stream": False
|
||
|
|
}
|
||
|
|
|
||
|
|
headers = {"Content-Type": "application/json"}
|
||
|
|
|
||
|
|
async with aiohttp.ClientSession() as session:
|
||
|
|
async with session.post(f"{globals.OLLAMA_URL}/api/generate", json=payload, headers=headers) as response:
|
||
|
|
if response.status == 200:
|
||
|
|
data = await response.json()
|
||
|
|
return data.get("response", "No description.")
|
||
|
|
else:
|
||
|
|
return f"Error: {response.status}"
|
||
|
|
|
||
|
|
async def rephrase_as_miku(qwen_output, user_prompt):
|
||
|
|
await switch_model(globals.OLLAMA_MODEL) # likely llama3
|
||
|
|
|
||
|
|
with open("miku_prompt.txt", "r", encoding="utf-8") as f:
|
||
|
|
system_prompt = f.read()
|
||
|
|
|
||
|
|
relevant_docs_lore = miku_vectorstore.similarity_search(qwen_output, k=3)
|
||
|
|
context = "\n\n".join([doc.page_content for doc in relevant_docs_lore])
|
||
|
|
|
||
|
|
full_prompt = (
|
||
|
|
f"{context}\n\n"
|
||
|
|
f"The user asked: \"{user_prompt}\"\n"
|
||
|
|
f"The image contains: \"{qwen_output}\"\n\n"
|
||
|
|
f"Respond like Miku: cheerful, helpful, and opinionated when asked.\n\n"
|
||
|
|
f"Miku is currently feeling: {globals.CURRENT_MOOD}\n Please respond in a way that reflects this emotional tone.\n\n"
|
||
|
|
f"Miku:"
|
||
|
|
)
|
||
|
|
|
||
|
|
payload = {
|
||
|
|
"model": globals.OLLAMA_MODEL,
|
||
|
|
"prompt": full_prompt,
|
||
|
|
"system": system_prompt,
|
||
|
|
"stream": False
|
||
|
|
}
|
||
|
|
|
||
|
|
headers = {"Content-Type": "application/json"}
|
||
|
|
|
||
|
|
async with aiohttp.ClientSession() as session:
|
||
|
|
async with session.post(f"{globals.OLLAMA_URL}/api/generate", json=payload, headers=headers) as response:
|
||
|
|
if response.status == 200:
|
||
|
|
data = await response.json()
|
||
|
|
return data.get("response", "No response.")
|
||
|
|
else:
|
||
|
|
return f"Error: {response.status}"
|