Category: CODES

mechanize downloaded file, extract perl:

#!/usr/bin/perl
use strict;
use warnings;
use HTML::TokeParser;

# Define the filename containing the list of HTML files
my $filename = "file_names.txt";
my $output_file = "extract.txt";

# Open the file containing the list of HTML files
open my $file_list_fh, '<', $filename or die "Cannot open file $filename: $!";
# Open the output file for writing
open my $output_fh, '>', $output_file or die "Cannot open file $output_file: $!";

# Loop through each line of the file
while (my $html_file = <$file_list_fh>) {
    chomp $html_file; # Remove newline character
    next if $html_file =~ /^\s*$/; # Skip empty lines

    # Read the entire HTML file into a string
    open my $html_fh, '<', $html_file or die "Cannot open HTML file $html_file: $!";
    my $html_content = do { local $/; <$html_fh> };
    close $html_fh;

    # Replace <br> tags with semicolons
    $html_content =~ s/<br>/;/g;

    # Now, process the modified HTML content using HTML::TokeParser and extract table values
    my $parser = HTML::TokeParser->new(\$html_content) or die "Cannot create HTML::TokeParser object: $!";
    my ($inside_td) = 0;
    my ($extracted_content, $old_content) = ("", "");

    while (my $token = $parser->get_token) {
        if ($token->[0] eq 'S' && $token->[1] eq 'td' && defined $token->[2]{valign} && $token->[2]{valign} eq 'top') {
            # Start of a table cell
            $inside_td = 1;
        } elsif ($token->[0] eq 'E' && $token->[1] eq 'td') {
            # End of a table cell
            $inside_td = 0;
            if ($extracted_content || $old_content) {
                print $output_fh "$html_file;$extracted_content;$old_content\n"; # Write the filename, extracted content, and old content to the output file if any exists
                ($extracted_content, $old_content) = ("", ""); # Reset the extracted content and old content for the next cell
            }
        } elsif ($token->[0] eq 'T' && $inside_td) {
            # Text token inside a table cell
            $old_content .= $token->[1]; # Accumulate old content if inside <td>
        }
    }
}

# Close the files
close $file_list_fh;
close $output_fh;

extract_TEL.pl (removes leading space of the line having TEL:)

#!/usr/bin/perl
use strict;
use warnings;

# Define the input and output filenames
my $input_filename = 'extract.txt';
my $output_filename = 'extract_TEL.txt';

# Open the input file for reading
open(my $fh_in, '<', $input_filename) or die "Could not open file '$input_filename' $!";
# Open the output file for writing
open(my $fh_out, '>', $output_filename) or die "Could not open file '$output_filename' for writing: $!";

# Loop through each line of the input file
while (my $line = <$fh_in>) {
    chomp $line;  # Remove newline character

    # Remove leading spaces from the line if it contains "TEL:"
    $line =~ s/^\s+// if $line =~ /^\s*TEL:/;

    print $fh_out "$line\n";  # Print the modified line to the output file
}

# Close the file handles
close $fh_in;
close $fh_out;

print "Processing complete. Results saved in $output_filename\n";

extract_TEL_single_line.pl (move TEL: line to above line)

#!/usr/bin/perl
use strict;
use warnings;

# Define the input and output filenames
my $input_filename = 'extract_TEL.txt';
my $output_filename = 'extract_TEL_single_line.txt';

# Open the input file for reading
open(my $fh_in, '<', $input_filename) or die "Could not open file '$input_filename' $!";
# Open the output file for writing
open(my $fh_out, '>', $output_filename) or die "Could not open file '$output_filename' for writing: $!";

my $prev_line = '';  # Initialize a variable to store the previous line

# Loop through each line of the input file
while (my $line = <$fh_in>) {
    chomp $line;  # Remove newline character

    if ($line =~ /^TEL:/) {
        # If the line starts with "TEL:", concatenate it with the previous line
        $prev_line =~ s/\s+$//;  # Remove trailing whitespace from the previous line
        $line =~ s/^\s+//;  # Remove leading whitespace from the current line
        print $fh_out "$prev_line $line\n";
    } else {
        # If the line does not start with "TEL:", print it as is
        print $fh_out "$line\n";
        $prev_line = $line;  # Update the previous line
    }
}

# Close the file handles
close $fh_in;
close $fh_out;

print "Processing complete. Results saved in $output_filename\n";

extract_TEL_single_line_only.pl (only lines having TEL: on it)

#!/usr/bin/perl
use strict;
use warnings;

# Define the input and output filenames
my $input_filename = 'extract_TEL_single_line.txt';
my $output_filename = 'extract_TEL_single_line_only.txt';

# Open the input file for reading
open(my $fh_in, '<', $input_filename) or die "Could not open file '$input_filename' $!";
# Open the output file for writing
open(my $fh_out, '>', $output_filename) or die "Could not open file '$output_filename' for writing: $!";

# Loop through each line of the input file
while (my $line = <$fh_in>) {
    chomp $line;  # Remove newline character

    if ($line =~ /TEL:/) {
        # If the line contains "TEL:", print it to the output file
        print $fh_out "$line\n";
    }
}

# Close the file handles
close $fh_in;
close $fh_out;

print "Processing complete. Lines containing 'TEL:' extracted to $output_filename\n";

extract_TEL_single_line_distinct.pl (distinct lines only)

#!/usr/bin/perl
use strict;
use warnings;

# Define the input and output filenames
my $input_filename = 'extract_TEL_single_line_only.txt';
my $output_filename = 'extract_TEL_single_line_distinct.txt';

# Open the input file for reading
open(my $fh_in, '<', $input_filename) or die "Could not open file '$input_filename' $!";
# Open the output file for writing
open(my $fh_out, '>', $output_filename) or die "Could not open file '$output_filename' for writing: $!";

# Define a hash to store unique lines
my %unique_lines;

# Loop through each line of the input file
while (my $line = <$fh_in>) {
    chomp $line;  # Remove newline character

    if ($line =~ /TEL:/) {
        # If the line contains "TEL:", add it to the hash
        $unique_lines{$line} = 1;
    }
}

# Write the unique lines to the output file
foreach my $line (keys %unique_lines) {
    print $fh_out "$line\n";
}

# Close the file handles
close $fh_in;
close $fh_out;

print "Processing complete. Distinct lines containing 'TEL:' extracted to $output_filename\n";

extract_TEL_single_line_distinct_semicolon.pl (replace | with 😉

#!/usr/bin/perl
use strict;
use warnings;

# Define the input and output filenames
my $input_filename = 'extract_TEL_single_line_distinct.txt';
my $output_filename = 'extract_TEL_single_line_distinct_semicolon.txt';

# Open the input file for reading
open(my $fh_in, '<', $input_filename) or die "Could not open file '$input_filename' $!";
# Open the output file for writing
open(my $fh_out, '>', $output_filename) or die "Could not open file '$output_filename' for writing: $!";

# Loop through each line of the input file
while (my $line = <$fh_in>) {
    chomp $line;  # Remove newline character

    # Replace '|' with ';'
    $line =~ s/\|/;/g;

    # Write the modified line to the output file
    print $fh_out "$line\n";
}

# Close the file handles
close $fh_in;
close $fh_out;

print "Processing complete. '|' replaced with ';' in $output_filename\n";

filename generation to use for extract.pl

#!/usr/bin/perl
use strict;
use warnings;

# Open the output file for writing
open my $output_fh, '>', "file_names.txt" or die "Cannot open file file_names.txt for writing: $!";

# Generate file names from 1.html to 481.html and write them to the output file
for my $i (1..1320) {
    my $file_name = "$i.html";
    print $output_fh "$file_name\n";
}

# Close the file
close $output_fh;

print "File names generated and saved to file_names.txt\n";

using below code based extracted URL, use below Perl Mechanize to download all pages of each business:

#!/usr/bin/perl

use strict;
use warnings;
use Encode;  # To handle encoding issues
use WWW::Mechanize;

# Read URLs from the text file
my $filename = 'urls.txt';

# Create a new WWW::Mechanize object
my $mech = WWW::Mechanize->new();

# Open the text file
open(my $fh, '<', $filename) or die "Could not open file '$filename' $!";

# Initialize page number
my $page_num = 1;

# Loop through each URL in the file
while (my $url = <$fh>) {
    chomp $url;  # Remove newline character

    # Visit the URL
    $mech->get($url);

    # Get the content of the current page
    my $content = $mech->content();

    # Save the content to a file with EUC-KR encoding
    my $filename = sprintf("%d.html", $page_num);
    open(my $fh_out, '>:encoding(EUC-KR)', $filename) or die "Could not open file '$filename' for writing: $!";
    print $fh_out $content;
    close $fh_out;

    # Find pagination links and click on them to navigate through pages
    my @pagination_links = $mech->find_all_links(url_regex => qr/page=/i);
    for my $link (@pagination_links) {
        $mech->get($link->url);
        my $content = $mech->content();
        $page_num++;
        my $filename = sprintf("%d.html", $page_num);
        open(my $fh_out, '>:encoding(EUC-KR)', $filename) or die "Could not open file '$filename' for writing: $!";
        print $fh_out $content;
        close $fh_out;
    }

    # Increment page number
    $page_num++;
}

# Close the file handle
close($fh);

download as txt file using Firefox and use below code to extract each business URL:

#!/usr/bin/perl

use strict;
use warnings;

# Define the output file name
my $output_file = 'KoreaDaily.txt';

# Open the output file for writing
open(my $fh_out, '>', $output_file) or die "Could not open file '$output_file' for writing: $!";

# Loop through files Atlanta_01.txt to Atlanta_13.txt
for my $file_number (1..14) {
    my $input_file = "Atlanta_" . sprintf("%02d", $file_number) . ".txt";
    
    # Open the current input file for reading
    open(my $fh_in, '<', $input_file) or die "Could not open file '$input_file': $!";

    # Iterate through each line of the current input file
    while (my $line = <$fh_in>) {
        # Check if the line contains 'cat_code='
        if ($line =~ /cat_code=/) {
            # Write the line to the output file
            print $fh_out $line;
        }
    }

    # Close the current input file handle
    close($fh_in);
}

# Close the output file handle
close($fh_out);

print "Extraction completed. Results saved in '$output_file'.\n";

/* below CSS hides post titles */

.entry-title{display: none;}

/* below CSS hides shopping cart logo beside menu */

.et-cart-info{display:none;}

/* below CSS displays secondary menu on smartphone */

@media screen and (max-width: 980px) {
	#page-container {
		padding-top: 0px !important;
	}
	#top-header {
		position: static !important;
		display: block !important;
	}
	#top-header .container {
		padding-top: 0.75em !important;
	}
	#page-container #main-header {
		position: relative !important;
		top: 0 !important;
	}
	#top-header #et-secondary-menu,
	#top-header #et-secondary-nav {
		display: block !important;
	}
	#top-header #et-secondary-nav li {
		text-align: center !important;
	}
	#et-main-area {
		padding-top: 1px;
	}
}

use strict;
use warnings;
use XML::LibXML;
use Text::CSV;

# Replace 'your_xml_file.xml' with the path to your XML file
my $xml_file = 'title.xml';

# Create an XML::LibXML parser
my $parser = XML::LibXML->new;

# Parse the XML file
my $doc = $parser->parse_file($xml_file);

# Find all <title> tags in the XML
my @titles = $doc->findnodes('//title');

# Create a Text::CSV object to write to a CSV file
my $csv = Text::CSV->new({ binary => 1, eol => "\n" });

# Open the CSV file for writing
open my $csv_fh, '>', 'oriental_medicine_seoul.csv' or die "Failed to open CSV file: $!";

# Write header row to the CSV file
$csv->print($csv_fh, ["Title"]);

# Extract and write the text within each <title> tag to the CSV file
foreach my $title (@titles) {
    my $title_text = $title->to_literal;
    $csv->print($csv_fh, [$title_text]);
}

# Close the CSV file
close $csv_fh;

# If no <title> tags are found
if (!@titles) {
    print "No title tags found in the XML.\n";
} else {
    print "Extraction complete. Titles saved to 'oriental_medicine_seoul.csv'.\n";
}

business_type_update.pl

#!/usr/bin/perl

use strict;
use warnings;

# Input file
my $input_file = 'business_type_original.txt';

# Output file
my $output_file = 'business_type_replaced.txt';

# Open the input file for reading
open my $input_fh, '<', $input_file or die "Cannot open input file '$input_file': $!";

# Open the output file for writing
open my $output_fh, '>', $output_file or die "Cannot open output file '$output_file': $!";

# Process each line of the input file
while (my $line = <$input_fh>) {
    # Replace "이삿짐" with "치과" in the line
    $line =~ s/이삿짐/회계 사무실/g;

    # Write the modified line to the output file
    print $output_fh $line;
}

# Close the input and output files
close $input_fh;
close $output_fh;

print "Replacement complete. Results saved in '$output_file'\n";

sqlitecodes.txt

select '"' || field1 || '",' from business_type_replaced order by field1 desc

moving_seoul.html

<!DOCTYPE html>
<html>
<head>
    <title>Open Search Words in Reverse Order</title>
</head>
<body>
    <script>
        var searchWords = [
"이삿짐 중랑구 서울특별시",
"이삿짐 중구 서울특별시",
"이삿짐 종로구 서울특별시",
"이삿짐 은평구 서울특별시",
"이삿짐 용산구 서울특별시",
"이삿짐 영등포구 서울특별시",
"이삿짐 양천구 서울특별시",
"이삿짐 송파구 서울특별시",
"이삿짐 성북구 서울특별시",
"이삿짐 성동구 서울특별시",
"이삿짐 서초구 서울특별시",
"이삿짐 서대문구 서울특별시",
"이삿짐 마포구 서울특별시",
"이삿짐 동작구 서울특별시",
"이삿짐 동대문구 서울특별시",
"이삿짐 도봉구 서울특별시",
"이삿짐 노원구 서울특별시",
"이삿짐 금천구 서울특별시",
"이삿짐 구로구 서울특별시",
"이삿짐 광진구 서울특별시",
"이삿짐 관악구 서울특별시",
"이삿짐 강서구 서울특별시",
"이삿짐 강북구 서울특별시",
"이삿짐 강동구 서울특별시",
"이삿짐 강남구 서울특별시"
        ];

        for (var i = 0; i < Math.min(25, searchWords.length); i++) {
            window.open("https://www.google.com/search?q=" + encodeURIComponent(searchWords[i]), "_blank");
        }
    </script>
</body>
</html>

firefox_02.pl

use strict;
use warnings;

# Initialize an array to store extracted content
my @extracted_content;

# Process each HTML file (1.html to 25.html)
for my $file_number (1..25) {
    my $input_file = "${file_number}.html";

    # Open the current input file for reading
    open my $input_fh, '<', $input_file or warn "Cannot open input file '$input_file': $!";

    # Read each line and look for URLs and +82 content
    while (my $line = <$input_fh>) {
        chomp $line;

        # Extract URLs matching /https:\/\/www\.google\.com\/maps\/place/
        if ($line =~ /https:\/\/www.google.com\/maps\/place/) {
            # Remove leading '<' and trailing '>'
            $line =~ s/^<(.+)>$/$1/;
            push @extracted_content, "$input_file\t$line";  # Use a tab character as the delimiter
        }

        # Extract content starting with +82 and going to the end of the line
        if ($line =~ /\+82(.+)/) {
            my $plus82_content = "+82$1";  # Include +82 in the extracted content
            push @extracted_content, "$input_file\t$plus82_content";  # Use a tab character as the delimiter
        }
    }

    # Close the current input file
    close $input_fh;
}

# Save the extracted content to a file
my $output_file = 'firefox_02.txt';
open my $output_fh, '>', $output_file or die "Cannot open output file '$output_file': $!";
foreach my $entry (@extracted_content) {
    print $output_fh "$entry\n";
}
close $output_fh;

print "Extraction complete. Extracted content is saved in '$output_file'\n";

firefox_website_02.pl

use strict;
use warnings;

# Initialize variables to store extracted URLs
my @google_maps_urls;

# Process each HTML file (1.html to 25.html)
for my $file_number (1..25) {
    my $input_file = "${file_number}.html";

    # Open the current input file for reading
    open my $input_fh, '<', $input_file or warn "Cannot open input file '$input_file': $!";

    my $previous_line = '';
    my $current_line = '';
    my $extracting = 0;

    # Read each line and look for URLs
    while (my $line = <$input_fh>) {
        chomp $line;

        # Check if the current line contains "+82"
        if ($current_line =~ /\+82/) {
            # Check if the next line contains "Website"
            if ($line =~ /Website/) {
                $extracting = 1;
                push @google_maps_urls, "$input_file, $current_line";
            }
        }

        # Continue extracting 3 more lines if we're in an extraction state
        if ($extracting) {
            push @google_maps_urls, "$input_file, $line";
            if (@google_maps_urls % 4 == 0) {
                $extracting = 0;
            }
        }

        # Store the current line for the next iteration
        $previous_line = $current_line;
        $current_line = $line;
    }

    # Close the current input file
    close $input_fh;
}

# Save the extracted URLs to a file
my $output_file = 'firefox_website_02.txt';
open my $output_fh, '>', $output_file or die "Cannot open output file '$output_file': $!";

foreach my $url (@google_maps_urls) {
    print $output_fh "$url\n";
}

# Close the output file
close $output_fh;

print "Extraction complete. Extracted 4 lines (including +82 and Website) are saved in '$output_file'\n";

sqlitecodes.txt

82.txt

SELECT
    field1, field2,
    CASE
        WHEN INSTR(field2, '+82') > 0 THEN
            SUBSTR(field2, INSTR(field2, '+82'))
        ELSE
            field2
    END AS extracted_value
FROM firefox_website_02;

http.txt

SELECT
    extracted_value, field1, field2,
    CASE
        WHEN field2 LIKE '<%' AND field2 LIKE '%>' THEN
            SUBSTR(field2, INSTR(field2, '<') + 1, INSTR(field2, '>') - INSTR(field2, '<') - 1)
        WHEN field2 LIKE '<%' THEN
            'Starts with <, but missing >'
        ELSE
            'Does not start with <'
    END AS extracted_value
FROM firefox_website_02_PHONE;

SELECT
    extracted_value, field2,
    CASE
        WHEN field2 LIKE '<%' AND field2 LIKE '%>' THEN
            SUBSTR(field2, INSTR(field2, '<') + 1, INSTR(field2, '>') - INSTR(field2, '<') - 1)
        WHEN field2 LIKE '<%' THEN
            'Starts with <, but missing >'
        ELSE
            'Does not start with <'
    END AS extracted_value
FROM firefox_website_02_PHONE;