PHP 시작하기 - curl이용 웹크롤링

PHP에서 curl를 이용해서 웹크롤링 해보자.

 

예로 Naver 배당주 페이지를 웹크롤링해보자.

 

※ 소스

<?php

require_once 'simple_html_dom.php';

 

$cnt = 0;

$fp = fopen("배당주.txt", 'w');

 

fwrite($fp, "[");  

$agent = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)';

for($i = 1;$i < 27;$i++) {

    $html ="";

    $data["page"] = $i;

    $url = 'https://finance.naver.com/sise/dividend_list.naver?field=dividend_rate&sosok=&ordering=desc&page=' . $i;

    echo($url .'<br>');

    $ch[$i] = curl_init();    //curl 초기화

    curl_setopt($ch[$i], CURLOPT_URL, $url);     //URL 지정하기

    curl_setopt($ch[$i], CURLOPT_HTTPHEADER, array("Cache-Control: no-cache"));

    curl_setopt ($ch[$i], CURLOPT_HEADER,          0);

    curl_setopt ($ch[$i], CURLOPT_RETURNTRANSFER,  1);

    curl_setopt ($ch[$i], CURLOPT_POST,            0);

    curl_setopt ($ch[$i], CURLOPT_USERAGENT,       $agent);

    curl_setopt ($ch[$i], CURLOPT_REFERER,         "");

    curl_setopt ($ch[$i], CURLOPT_TIMEOUT,         3);

    $response[$i] = curl_exec($ch[$i]);

    curl_close($ch[$i]);

    //echo($response);

    $html = str_get_html($response[$i]);

    foreach( $html->find( 'table.type_1 tr td a' ) as $a ) {

        $cnt ++;

        if ($cnt > 1) {

            fwrite($fp, ",");

            //echo ',';

        }

        fwrite($fp, '["'

            . str_replace("code=","",strstr($a->href,"code="))

            . '","'

            . iconv("EUC-KR", "UTF-8", $a->plaintext)

            . '"]');  

        echo '{"code":"';

        echo str_replace("code=","",strstr($a->href,"code="));

    }

    $html->clear();

}

fwrite($fp, "]");

fclose($fp);

?>

 

※ 실행