Python NLTK Tutorial: Natural Language Processing

# Install NLTK using pip
pip install nltk

# If you're using Python 3 specifically, you may need:
pip3 install nltk

# Download required NLTK data
import nltk

# Download all essential packages (recommended for beginners)
nltk.download('popular')

# OR download specific packages individually:
# nltk.download('punkt')        # For tokenization
# nltk.download('averaged_perceptron_tagger')  # For POS tagging
# nltk.download('maxent_ne_chunker')  # For named entity recognition
# nltk.download('words')        # English word list
# nltk.download('stopwords')    # Common words to filter out
# nltk.download('vader_lexicon')  # For sentiment analysis

import nltk
from nltk.tokenize import word_tokenize

# Sample text
text = "Natural language processing is fascinating! It helps computers understand human language."

# Tokenize into words
tokens = word_tokenize(text)

# Display results
print("Original text:")
print(text)
print("\nTokenized words:")
print(tokens)
print(f"\nTotal number of tokens: {len(tokens)}")

from nltk.tokenize import sent_tokenize

# Text with multiple sentences
paragraph = "AI is transforming education. Students can now learn at their own pace. Teachers have powerful new tools to help them."

# Tokenize into sentences
sentences = sent_tokenize(paragraph)

# Display results
print("Sentences found:")
for i, sentence in enumerate(sentences, 1):
    print(f"{i}. {sentence}")

import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag

# Sample sentence
sentence = "The quick brown fox jumps over the lazy dog."

# Tokenize and tag parts of speech
tokens = word_tokenize(sentence)
tagged = pos_tag(tokens)

# Display results
print("Word → Part of Speech")
print("-" * 30)
for word, tag in tagged:
    print(f"{word:15} → {tag}")

# Count different types
print("\n" + "="*30)
nouns = [word for word, tag in tagged if tag.startswith('NN')]
verbs = [word for word, tag in tagged if tag.startswith('VB')]
adjectives = [word for word, tag in tagged if tag.startswith('JJ')]

print(f"Nouns: {nouns}")
print(f"Verbs: {verbs}")
print(f"Adjectives: {adjectives}")

import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag, ne_chunk

# Sample text with named entities
text = "Apple Inc. was founded by Steve Jobs in Cupertino, California. Microsoft was started by Bill Gates in Seattle."

# Process the text
tokens = word_tokenize(text)
tagged = pos_tag(tokens)
entities = ne_chunk(tagged)

# Display the tree structure
print("Named Entity Tree:")
print(entities)

# Extract named entities
print("\nExtracted Named Entities:")
print("-" * 40)
for chunk in entities:
    if hasattr(chunk, 'label'):
        entity_name = ' '.join(c[0] for c in chunk)
        entity_type = chunk.label()
        print(f"{entity_name:25} → {entity_type}")

# Alternative: More detailed extraction
def extract_entities(text):
    """Extract and categorize named entities from text."""
    tokens = word_tokenize(text)
    tagged = pos_tag(tokens)
    entities = ne_chunk(tagged)
    
    results = {'PERSON': [], 'ORGANIZATION': [], 'GPE': [], 'OTHER': []}
    
    for chunk in entities:
        if hasattr(chunk, 'label'):
            entity_name = ' '.join(c[0] for c in chunk)
            label = chunk.label()
            
            if label in results:
                results[label].append(entity_name)
            else:
                results['OTHER'].append(entity_name)
    
    return results

# Use the function
entities_dict = extract_entities(text)
print("\n" + "="*40)
print("Categorized Entities:")
print(f"People: {entities_dict['PERSON']}")
print(f"Organizations: {entities_dict['ORGANIZATION']}")
print(f"Locations (GPE): {entities_dict['GPE']}")

from nltk.sentiment import SentimentIntensityAnalyzer

# Create sentiment analyzer
sia = SentimentIntensityAnalyzer()

# Test different sentences
sentences = [
    "I love this product! It's absolutely amazing!",
    "This is the worst experience ever. Very disappointed.",
    "The movie was okay. Nothing special.",
    "I'm feeling pretty good about the test results.",
    "Ugh, another rainy Monday. Just great. 😒"
]

print("Sentiment Analysis Results")
print("="*60)

for sentence in sentences:
    # Get sentiment scores
    scores = sia.polarity_scores(sentence)
    
    # Determine overall sentiment
    if scores['compound'] >= 0.05:
        sentiment = "POSITIVE ✓"
    elif scores['compound'] <= -0.05:
        sentiment = "NEGATIVE ✗"
    else:
        sentiment = "NEUTRAL —"
    
    print(f"\nText: {sentence}")
    print(f"  Positive: {scores['pos']:.2f}")
    print(f"  Negative: {scores['neg']:.2f}")
    print(f"  Neutral:  {scores['neu']:.2f}")
    print(f"  Compound: {scores['compound']:.2f}")
    print(f"  Overall:  {sentiment}")

# Analyzing longer text
def analyze_review(review_text):
    """Analyze a product review or longer text."""
    scores = sia.polarity_scores(review_text)
    compound = scores['compound']
    
    # Determine rating
    if compound >= 0.5:
        rating = "⭐⭐⭐⭐⭐ Highly Positive"
    elif compound >= 0.05:
        rating = "⭐⭐⭐⭐ Positive"
    elif compound <= -0.5:
        rating = "⭐ Highly Negative"
    elif compound <= -0.05:
        rating = "⭐⭐ Negative"
    else:
        rating = "⭐⭐⭐ Neutral"
    
    return {
        'scores': scores,
        'rating': rating,
        'confidence': abs(compound)
    }

# Example review analysis
review = """
This product exceeded my expectations! The quality is fantastic and 
it arrived quickly. Customer service was helpful when I had questions. 
Highly recommend to anyone looking for a reliable option.
"""

result = analyze_review(review)
print("\n" + "="*60)
print("Review Analysis:")
print(f"Rating: {result['rating']}")
print(f"Confidence: {result['confidence']:.2%}")

import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from collections import Counter

# Download stopwords if not already done
# nltk.download('stopwords')

# Sample text
text = """
Artificial intelligence is transforming education. Teachers can use AI to 
personalize learning for students. Students can learn at their own pace. 
AI systems can provide immediate feedback. The future of education looks 
very different with AI technology.
"""

# Tokenize and convert to lowercase
tokens = word_tokenize(text.lower())

# Get English stopwords
stop_words = set(stopwords.words('english'))

# Filter out stopwords and punctuation
filtered_words = [
    word for word in tokens 
    if word.isalnum() and word not in stop_words
]

print("Original tokens:", len(tokens))
print("After filtering:", len(filtered_words))
print("\nFiltered words:", filtered_words[:20])  # Show first 20

# Count word frequency
word_freq = Counter(filtered_words)

print("\n" + "="*50)
print("Top 10 Most Common Words:")
print("="*50)
for word, count in word_freq.most_common(10):
    print(f"{word:20} → {count} times")

# Find keywords (words appearing 2+ times)
keywords = [word for word, count in word_freq.items() if count >= 2]
print(f"\nKeywords (appearing 2+ times): {keywords}")

# Calculate basic statistics
unique_words = len(word_freq)
total_words = len(filtered_words)
vocabulary_richness = unique_words / total_words

print(f"\nText Statistics:")
print(f"  Unique words: {unique_words}")
print(f"  Total words: {total_words}")
print(f"  Vocabulary richness: {vocabulary_richness:.2%}")