ChatGPT如何构建可以回答有关您网站的问题的 AI

入门

• python -m venv env
• 
  
• source env/bin/activate
• 
  
• pip install -r requirements.txt

1. def remove_newlines(serie):
2. serie = serie.str.replace('n', ' ')
3. serie = serie.str.replace('n', ' ')
4. serie = serie.str.replace('  ', ' ')
5. serie = serie.str.replace('  ', ' ')
6. return serie

1. import pandas as pd
2. 
   
3. # Create a list to store the text files
4. texts=[]
5. 
   
6. # Get all the text files in the text directory
7. for file in os.listdir("text/" + domain + "/"):
8. 
   
9. # Open the file and read the text
10. with open("text/" + domain + "/" + file, "r", encoding="UTF-8") as f:
11. text = f.read()
12. 
    
13. # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.
14. texts.append((file[11:-4].replace('-',' ').replace('_', ' ').replace('#update',''), text))
15. 
    
16. # Create a dataframe from the list of texts
17. df = pd.DataFrame(texts, columns = ['fname', 'text'])
18. 
    
19. # Set the text column to be the raw text with the newlines removed
20. df['text'] = df.fname + ". " + remove_newlines(df.text)
21. df.to_csv('processed/scraped.csv')
22. df.head()

• 一个有用的经验法则是，对于普通英文文本，一个标记通常对应于 ~4 个字符的文本。这相当于大约 ¾ 个单词（因此 100 个标记 ~= 75 个单词）。

1. import tiktoken
2. 
   
3. # Load the cl100k_base tokenizer which is designed to work with the ada-002 model
4. tokenizer = tiktoken.get_encoding("cl100k_base")
5. 
   
6. df = pd.read_csv('processed/scraped.csv', index_col=0)
7. df.columns = ['title', 'text']
8. 
   
9. # Tokenize the text and save the number of tokens to a new column
10. df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))
11. 
    
12. # Visualize the distribution of the number of tokens per row using a histogram
13. df.n_tokens.hist()

1. max_tokens = 500
2. 
   
3. # Function to split the text into chunks of a maximum number of tokens
4. def split_into_many(text, max_tokens = max_tokens):
5. 
   
6. # Split the text into sentences
7. sentences = text.split('. ')
8. 
   
9. # Get the number of tokens for each sentence
10. n_tokens = [len(tokenizer.encode(" " + sentence)) for sentence in sentences]
11. 
    
12. chunks = []
13. tokens_so_far = 0
14. chunk = []
15. 
    
16. # Loop through the sentences and tokens joined together in a tuple
17. for sentence, token in zip(sentences, n_tokens):
18. 
    
19. # If the number of tokens so far plus the number of tokens in the current sentence is greater
20. # than the max number of tokens, then add the chunk to the list of chunks and reset
21. # the chunk and tokens so far
22. if tokens_so_far + token > max_tokens:
23. chunks.append(". ".join(chunk) + ".")
24. chunk = []
25. tokens_so_far = 0
26. 
    
27. # If the number of tokens in the current sentence is greater than the max number of
28. # tokens, go to the next sentence
29. if token > max_tokens:
30. continue
31. 
    
32. # Otherwise, add the sentence to the chunk and add the number of tokens to the total
33. chunk.append(sentence)
34. tokens_so_far += token + 1
35. 
    
36. return chunks
37. 
    
38. 
    
39. shortened = []
40. 
    
41. # Loop through the dataframe
42. for row in df.iterrows():
43. 
    
44. # If the text is None, go to the next row
45. if row[1]['text'] is None:
46. continue
47. 
    
48. # If the number of tokens is greater than the max number of tokens, split the text into chunks
49. if row[1]['n_tokens'] > max_tokens:
50. shortened += split_into_many(row[1]['text'])
51. 
    
52. # Otherwise, add the text to the list of shortened texts
53. else:
54. shortened.append( row[1]['text'] )

1. df = pd.DataFrame(shortened, columns = ['text'])
2. df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))
3. df.n_tokens.hist()

1. import openai
2. 
   
3. df['embeddings'] = df.text.apply(lambda x: openai.Embedding.create(input=x, engine='text-embedding-ada-002')['data'][0]['embedding'])
4. 
   
5. df.to_csv('processed/embeddings.csv')
6. df.head()

使用嵌入构建问答系统

1. import numpy as np
2. from openai.embeddings_utils import distances_from_embeddings
3. 
   
4. df=pd.read_csv('processed/embeddings.csv', index_col=0)
5. df['embeddings'] = df['embeddings'].apply(eval).apply(np.array)
6. 
   
7. df.head()

1. def create_context(
2. question, df, max_len=1800, size="ada"
3. ):
4. """
5. Create a context for a question by finding the most similar context from the dataframe
6. """
7. 
   
8. # Get the embeddings for the question
9. q_embeddings = openai.Embedding.create(input=question, engine='text-embedding-ada-002')['data'][0]['embedding']
10. 
    
11. # Get the distances from the embeddings
12. df['distances'] = distances_from_embeddings(q_embeddings, df['embeddings'].values, distance_metric='cosine')
13. 
    
14. 
    
15. returns = []
16. cur_len = 0
17. 
    
18. # Sort by distance and add the text to the context until the context is too long
19. for i, row in df.sort_values('distances', ascending=True).iterrows():
20. 
    
21. # Add the length of the text to the current length
22. cur_len += row['n_tokens'] + 4
23. 
    
24. # If the context is too long, break
25. if cur_len > max_len:
26. break
27. 
    
28. # Else add it to the text that is being returned
29. returns.append(row["text"])
30. 
    
31. # Return the context
32. return "nn###nn".join(returns)

1. def answer_question(
2. df,
3. model="text-davinci-003",
4. question="Am I allowed to publish model outputs to Twitter, without a human review?",
5. max_len=1800,
6. size="ada",
7. debug=False,
8. max_tokens=150,
9. stop_sequence=None
10. ):
11. """
12. Answer a question based on the most similar context from the dataframe texts
13. """
14. context = create_context(
15. question,
16. df,
17. max_len=max_len,
18. size=size,
19. )
20. # If debug, print the raw model response
21. if debug:
22. print("Context:n" + context)
23. print("nn")
24. 
    
25. try:
26. # Create a completions using the question and context
27. response = openai.Completion.create(
28. prompt=f"Answer the question based on the context below, and if the question can't be answered based on the context, say "I don't know"nnContext: {context}nn---nnQuestion: {question}nAnswer:",
29. temperature=0,
30. max_tokens=max_tokens,
31. top_p=1,
32. frequency_penalty=0,
33. presence_penalty=0,
34. stop=stop_sequence,
35. model=model,
36. )
37. return response["choices"][0]["text"].strip()
38. except Exception as e:
39. print(e)
40. return ""

1. answer_question(df, question="What day is it?", debug=False)
2. 
   
3. answer_question(df, question="What is our newest embeddings model?")
4. 
   
5. answer_question(df, question="What is ChatGPT?")

1. "I don't know."
2. 
   
3. 'The newest embeddings model is text-embedding-ada-002.'
4. 
   
5. 'ChatGPT is a model trained to interact in a conversational way. It is able to answer followup questions, admit its mistakes, challenge incorrect premises, and reject inappropriate requests.'