Natural Language Processing: Making Computers Understand Human Language

import nltk
from nltk.tokenize import word_tokenize, sent_tokenize

text = "Natural Language Processing is fascinating! It enables computers to understand human language."

# Sentence tokenization
sentences = sent_tokenize(text)
print(sentences)  # ['Natural Language Processing is fascinating!', 'It enables computers to understand human language.']

# Word tokenization
words = word_tokenize(text)
print(words)       # ['Natural', 'Language', 'Processing', 'is', 'fascinating', '!', 'It', 'enables', 'computers', 'to', 'understand', 'human', 'language', '.']

import re
from nltk.corpus import stopwords

def preprocess_text(text):
    # Convert to lowercase
    text = text.lower()

    # Remove punctuation and numbers
    text = re.sub(r'[^\w\s]', '', text)
    text = re.sub(r'\d+', '', text)

    # Tokenize
    tokens = word_tokenize(text)

    # Remove stopwords
    stop_words = set(stopwords.words('english'))
    filtered_tokens = [word for word in tokens if word not in stop_words]

    return filtered_tokens

# Example usage
processed = preprocess_text("Hello! This is a sample text for NLP processing in 2024.")
print(processed)  # ['hello', 'sample', 'text', 'nlp', 'processing']

from nltk.stem import PorterStemmer, WordNetLemmatizer

# Initialize stemmer and lemmatizer
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

word = "running"

# Stemming (removes suffixes aggressively)
print(stemmer.stem(word))  # "run"

# Lemmatization (context-aware root finding)
print(lemmatizer.lemmatize(word, pos='v'))  # "run"

from sklearn.feature_extraction.text import CountVectorizer

documents = [
    "Natural language processing is amazing",
    "Machine learning helps computers learn",
    "AI and NLP work together beautifully"
]

# Create Bag of Words representation
vectorizer = CountVectorizer()
bow_matrix = vectorizer.fit_transform(documents)

print("Vocabulary:", vectorizer.get_feature_names_out())
print("BoW Matrix:\n", bow_matrix.toarray())

from sklearn.feature_extraction.text import TfidfVectorizer

# TF-IDF vectorization
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(documents)

print("TF-IDF Features:", tfidf_vectorizer.get_feature_names_out())
print("TF-IDF Matrix:\n", tfidf_matrix.toarray())

from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# Sample training data
texts = ["I love this movie", "This movie is great", "I hate this", "This is terrible"]
labels = [1, 1, 0, 0]  # 1 = positive, 0 = negative

# Vectorize texts
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(texts)

# Split and train
X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.5)
clf = MultinomialNB()
clf.fit(X_train, y_train)

# Predict
predictions = clf.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, predictions)}")

from sklearn.svm import SVC

# SVM with RBF kernel for text classification
svm_classifier = SVC(kernel='rbf', C=1.0)
svm_classifier.fit(X_train, y_train)

svm_predictions = svm_classifier.predict(X_test)
print(f"SVM Accuracy: {accuracy_score(y_test, svm_predictions)}")

import gensim
from gensim.models import Word2Vec

# Train Word2Vec on sample sentences
sentences = [
    ['natural', 'language', 'processing', 'is', 'amazing'],
    ['machine', 'learning', 'helps', 'computers', 'learn'],
    ['ai', 'and', 'nlp', 'work', 'together']
]

# Train Word2Vec model
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)

# Get word vector
vector = model.wv['language']
print("Vector for 'language':", vector[:10])  # First 10 dimensions

# Find similar words
similar = model.wv.most_similar('language', topn=3)
print("Words similar to 'language':", similar)

import tensorflow as tf
from tensorflow.keras.layers import SimpleRNN, Dense, Embedding
from tensorflow.keras.models import Sequential

def create_rnn_classifier(vocab_size, embedding_dim, max_length):
    model = Sequential([
        Embedding(vocab_size, embedding_dim, input_length=max_length),
        SimpleRNN(64),
        Dense(1, activation='sigmoid')  # Binary classification
    ])

    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    return model

# Example usage
vocab_size = 10000
embedding_dim = 100
max_length = 200

rnn_model = create_rnn_classifier(vocab_size, embedding_dim, max_length)
print(rnn_model.summary())

import torch
import torch.nn as nn

class SelfAttention(nn.Module):
    def __init__(self, embed_size, heads):
        super(SelfAttention, self).__init__()
        self.embed_size = embed_size
        self.heads = heads
        self.head_dim = embed_size // heads

        assert (self.head_dim * heads == embed_size), "Embedding size needs to be divisible by heads"

        self.values = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.keys = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.queries = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.fc_out = nn.Linear(heads * self.head_dim, embed_size)

    def forward(self, values, keys, query):
        N = query.shape[0]
        value_len, key_len, query_len = values.shape[1], keys.shape[1], query.shape[1]

        # Split embedding into self.heads pieces
        values = values.reshape(N, value_len, self.heads, self.head_dim)
        keys = keys.reshape(N, key_len, self.heads, self.head_dim)
        queries = query.reshape(N, query_len, self.heads, self.head_dim)

        # Attention mechanism
        energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
        attention = torch.softmax(energy / (self.embed_size ** (1/2)), dim=3)

        out = torch.einsum("nhql,nlhd->nqhd", [attention, values]).reshape(
            N, query_len, self.heads * self.head_dim
        )

        out = self.fc_out(out)
        return out

from transformers import BertTokenizer, BertForSequenceClassification
from transformers import pipeline

# Load pre-trained BERT model for sentiment analysis
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

# Or use pipeline for simpler inference
sentiment_analyzer = pipeline('sentiment-analysis')

def analyze_sentiment(text):
    result = sentiment_analyzer(text)
    return result[0]

# Example usage
text = "I absolutely love this product!"
sentiment = analyze_sentiment(text)
print(sentiment)  # {'label': 'POSITIVE', 'score': 0.9998}

from transformers import pipeline

# NER with pre-trained model
ner_pipeline = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english")

text = "Albert Einstein was born in Ulm, Germany in 1879."

ner_results = ner_pipeline(text)

for entity in ner_results:
    print(f"{entity['word']}: {entity['entity']} (confidence: {entity['score']:.3f})")

from transformers import pipeline

summarizer = pipeline("summarization", model="facebook/bart-large-cnn")

article = """
Natural Language Processing (NLP) is a subfield of artificial intelligence that focuses on the interaction
between computers and humans through natural language. The goal of NLP is to read, decipher, understand,
and make sense of human language in a manner that is valuable.

Modern NLP combines computational linguistics with statistical, machine learning, and deep learning models.
Key areas include machine translation, sentiment analysis, text classification, and conversational agents.
"""

summary = summarizer(article, max_length=50, min_length=20, do_sample=False)
print(summary[0]['summary_text'])

from transformers import pipeline

qa_pipeline = pipeline("question-answering", model="distilbert-base-uncased-distilled-squad")

context = """
Natural Language Processing (NLP) is a field of artificial intelligence that focuses on enabling computers
to understand and process human language. It combines linguistics, computer science, and machine learning
to create systems that can interpret, generate, and respond to human text.
"""

question = "What is NLP?"

result = qa_pipeline(question=question, context=context)
print(f"Answer: {result['answer']} (confidence: {result['score']:.3f})")