_mySocket === null) { $this->_mySocket = new MySocketLibrary(getenv('yamap.host.url')); } return $this->_mySocket; } final protected function getMyStorage(): mixed { if ($this->_myStorage === null) { $this->_myStorage = new MyStorageLibrary(getenv('yamap.storage.upload.path')); $this->_myStorage->setBoardsEntity($this->getBoardsLibrary()->getEntity()); $this->_myStorage->setUserEntity($this->getUserEntity()); } return $this->_myStorage; } public function getBoardsLibrary(): BoardsLibrary { if ($this->_boards_library === null) { $this->_boards_library = new BoardsLibrary( getenv('yamap.storage.board.name'), $this->getUserEntity() ); } return $this->_boards_library; } public function getBoardLibrary(): BoardLibrary { if ($this->_board_library === null) { $this->_board_library = new BoardLibrary( $this->getBoardsLibrary()->getEntity(), $this->getUserEntity() ); } return $this->_board_library; } public function getUserEntity(): UserEntity { if ($this->_user_entity === null) { throw new \Exception("사용자정보가 없습니다."); } return $this->_user_entity; } public function setUserEntity(UserEntity $_user_entity): void { $this->_user_entity = $_user_entity; } private function save(int $file_sequence, string $mediaType, string $file_name, string $content): void { $this->getMyStorage()->setOriginName($file_name); $this->getMyStorage()->setOriginContent($content); $this->getMyStorage()->setOriginType($mediaType); $this->getMyStorage()->setOriginSequence($file_sequence); $this->_file_librarys[] = $this->getMyStorage()->save(); } //Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다. private function download(string $mediaType, string $url): array { $file_names = explode('/', $url); if (!is_array($file_names) || !count($file_names)) { throw new \Exception("URL이 파일명 형식이 아닙니다 : " . $this->getMySocket()->getHost() . $url); } $file_name = array_pop($file_names); $temps = explode(".", $file_name); $file_ext = array_pop($temps); if (!$this->isFileType($file_ext, $mediaType)) { throw new \Exception("파일명 형식이 {$mediaType}가 아닙니다"); } $content = $this->getMySocket()->getContent($url); log_message("notice", "{$file_name} 파일이 다운로드되었습니다!"); return array($file_name, $content); } private function mediaContent(array $urls): void { $file_sequence = 1; $this->_file_librarys = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 foreach ($urls as $mediaType => $url) { try { list($file_name, $content) = $this->download($mediaType, $url); $this->save($file_sequence, $mediaType, $file_name, $content); $file_sequence++; log_message("notice", __FUNCTION__ . " OriginType->{$mediaType} 작업 완료"); } catch (\Exception $e) { log_message("warning", sprintf( "\n---%s,OriginType->%s 오류---\n%s\n-----------------------------------------\n", __FUNCTION__, $mediaType, $e->getMessage() )); } } if (!count($this->_file_librarys)) { throw new \Exception("Download된 Content가 없습니다."); } } //Yamap ViewPage의 이미지나영상데이터가 있으면 URL과MediaType을 가져온다 private function getUrlsByDetailPageMediaType(string $mediaType, Crawler $selector, array $options, array $urls = []): array { $selector->filter($options["tag"])->each( function (Crawler $node) use (&$mediaType, &$options, &$urls): void { log_message("debug", sprintf( "getNode->%s[%s]", $options["tag"], $node->attr($options['attr']) )); $urls[$mediaType] = $node->attr($options["attr"]); } ); return $urls; } private function detailPage(array $listInfo): array { $response = $this->getMySocket()->getContent($listInfo['detail_url']); $selector = $this->getSelector($response, getenv("yamap.view.content.tag")); $urls = $this->getUrlsByDetailPageMediaType("image", $selector, ["tag" => "img", "attr" => "src"]); $urls = $this->getUrlsByDetailPageMediaType("video", $selector, ["tag" => "video", "attr" => "src"], $urls); log_message("notice", sprintf("\n-----------%s 작업완료--------\n%s\n-----------------------\n", __FUNCTION__, var_export($urls, true))); return $urls; } private function mainPage(string $url): array { $listInfos = []; $response = $this->getMySocket()->getContent($url); $selector = $this->getSelector($response, getenv("yamap.list.tag")); //div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 // log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); $selector->filter(getenv("yamap.list.item.tag"))->each( function (Crawler $node) use (&$listInfos): void { //bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool $nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text(); $hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text(); $date = $node->filter(getenv("yamap.list.item.date.tag"))->text(); if ($nickname != getenv("yamap.list.item.nickname.except")) { //작성자가 "관리자"가 아니 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 $link_node = $node->filter(getenv("yamap.list.item.link.tag")); $detail_url = $link_node->attr("href"); $title = $link_node->children()->last()->text(); $listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit]; } } ); if (!count($listInfos)) { throw new \Exception("Target URL이 없습니다."); } log_message("notice", __FUNCTION__ . " 작업 완료"); return $listInfos; } public function execute(): void { if ($this->getDebug()) { $listInfos = [ 'title' => getenv("yamap.view.test.title"), 'nickname' => getenv("yamap.view.test.nickname"), 'detail_url' => getenv("yamap.view.test.url"), 'time' => date("Y-m-d H:i:s"), 'hit' => 1, ]; } else { $listInfos = $this->mainPage(getenv("yamap.list.url")); } //Limit가 0이면 $listInfos 갯수만큼 다하고, LIMIT 갯수 혹은 item의 갯수중 작은수만큼 한다. $max_limit = intval(getenv("yamap.list.max_limit")); if ($max_limit) { $max_limit = count($listInfos) <= $max_limit ? count($listInfos) : $max_limit; } else { $max_limit = count($listInfos); } $i = 1; foreach ($listInfos as $listInfo) { if ($i <= $max_limit) { try { log_message("notice", "게시물 {$i}번째 {$listInfo["nickname"]} 작업시작"); $this->mediaContent($this->detailPage($listInfo)); //File DB 및 Board DB 등록작업 $this->getBoardLibrary()->create($i, $listInfo, $this->_file_librarys); log_message("notice", "게시물 {$i}번째 {$listInfo["nickname"]} 작업완료."); $i++; } catch (\Exception $e) { log_message("debug", $e->getMessage()); } } } log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다."); } }