{"id":3591,"date":"2024-02-09T23:28:14","date_gmt":"2024-02-10T08:28:14","guid":{"rendered":"https:\/\/crm270s.abramjmeister.com\/?p=3591"},"modified":"2024-02-09T23:28:14","modified_gmt":"2024-02-10T08:28:14","slug":"koreadaily-yellowpage-mechanize-perl-code","status":"publish","type":"post","link":"https:\/\/crm270s.mycpamytax.com\/?p=3591","title":{"rendered":"KoreaDaily Yellowpage Mechanize Perl Code"},"content":{"rendered":"\n<p class=\"wp-block-paragraph\">using below code based extracted URL, use below Perl Mechanize to download all pages of each business:<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>#!\/usr\/bin\/perl\n\nuse strict;\nuse warnings;\nuse Encode;  # To handle encoding issues\nuse WWW::Mechanize;\n\n# Read URLs from the text file\nmy $filename = 'urls.txt';\n\n# Create a new WWW::Mechanize object\nmy $mech = WWW::Mechanize->new();\n\n# Open the text file\nopen(my $fh, '&lt;', $filename) or die \"Could not open file '$filename' $!\";\n\n# Initialize page number\nmy $page_num = 1;\n\n# Loop through each URL in the file\nwhile (my $url = &lt;$fh>) {\n    chomp $url;  # Remove newline character\n\n    # Visit the URL\n    $mech->get($url);\n\n    # Get the content of the current page\n    my $content = $mech->content();\n\n    # Save the content to a file with EUC-KR encoding\n    my $filename = sprintf(\"%d.html\", $page_num);\n    open(my $fh_out, '>:encoding(EUC-KR)', $filename) or die \"Could not open file '$filename' for writing: $!\";\n    print $fh_out $content;\n    close $fh_out;\n\n    # Find pagination links and click on them to navigate through pages\n    my @pagination_links = $mech->find_all_links(url_regex => qr\/page=\/i);\n    for my $link (@pagination_links) {\n        $mech->get($link->url);\n        my $content = $mech->content();\n        $page_num++;\n        my $filename = sprintf(\"%d.html\", $page_num);\n        open(my $fh_out, '>:encoding(EUC-KR)', $filename) or die \"Could not open file '$filename' for writing: $!\";\n        print $fh_out $content;\n        close $fh_out;\n    }\n\n    # Increment page number\n    $page_num++;\n}\n\n# Close the file handle\nclose($fh);\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>using below code based extracted URL, use below Perl Mechanize to download all pages of each business:<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_et_pb_use_builder":"","_et_pb_old_content":"","_et_gb_content_width":"","footnotes":""},"categories":[26],"tags":[],"class_list":["post-3591","post","type-post","status-publish","format-standard","hentry","category-codes"],"_links":{"self":[{"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=\/wp\/v2\/posts\/3591","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3591"}],"version-history":[{"count":0,"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=\/wp\/v2\/posts\/3591\/revisions"}],"wp:attachment":[{"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3591"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3591"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/crm270s.mycpamytax.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3591"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}