diff --git a/app/Controllers/Mangboard/CrawlerController.php b/app/Controllers/Mangboard/CrawlerController.php index d87088c..0b01280 100644 --- a/app/Controllers/Mangboard/CrawlerController.php +++ b/app/Controllers/Mangboard/CrawlerController.php @@ -3,20 +3,44 @@ namespace App\Controllers\Mangboard; use App\Controllers\CommonController; -use App\Libraries\Mangboard\User; use App\Entities\Mangboard\UserEntity; use App\Libraries\MyCrawler\YamapCrawler; use App\Libraries\MyCrawler\YamoonCrawler; +use App\Models\Mangboard\UserModel; class CrawlerController extends CommonController { - private $_user = null; - private function getUser() + private $_userModel = null; + public function getUserModel(): UserModel { - if ($this->_user === null) { - $this->_user = new User(); + if ($this->_user_model === null) { + return $this->_user_model = new UserModel(); } - return $this->_user; + return $this->_user_model; + } + public function login(string $host, string $id, string $password): bool|UserEntity + { + $user_entity = $this->getUserModel()->getEntityByID($id); + // $response = $this->getWebLibrary($host)->getResponse( + // $host . getenv("mangboard.login.url"), + // "post", + // [ + // 'form_params' => [ + // 'user_id' => $id, + // 'password' => $password, + // ], + // ] + // ); + // if ($response->getStatusCode() == 200) { + // $entity = $this->getUserModel()->getEntityByLoginCheck($id, $password); + // if ($entity === null) { + // throw new \Exception("{$id}는 회원이 아니거나 암호가 맞지 않습니다."); + // } + // } else { + // throw new \Exception("연결실패:" . $response->getStatusCode()); + // } + log_message("notice", "{$id}로 로그인 성공"); + return $user_entity; } public function yamap(string $category, string $id = "", string $debug = "false"): string { @@ -24,7 +48,7 @@ class CrawlerController extends CommonController $id = $id == "" ? getenv("mangboard.login.default.id") : $id; $password = getenv("mangboard.login.default.password"); //1. 사이트 로그인 처리 - $user_entity = $this->getUser()->login(getenv("mangboard.host.url"), $id, $password); + $user_entity = $this->login(getenv("mangboard.host.url"), $id, $password); //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. $crawler = new YamapCrawler(getenv('yamap.host.url'), $category, $user_entity); $crawler->setDebug($debug === "true" ? true : false); @@ -41,7 +65,7 @@ class CrawlerController extends CommonController $id = $id == "" ? getenv("mangboard.login.default.id") : $id; $password = getenv("mangboard.login.default.password"); //1. 사이트 로그인 처리 - $user_entity = $this->getUser()->login(getenv("mangboard.host.url"), $id, $password); + $user_entity = $this->login(getenv("mangboard.host.url"), $id, $password); //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. $crawler = new YamoonCrawler(getenv("yamoon.host.url"), $category, $user_entity); $crawler->setDebug($debug === "true" ? true : false); diff --git a/app/Libraries/Mangboard/Board.php b/app/Libraries/Mangboard/Board.php deleted file mode 100644 index c85f6fc..0000000 --- a/app/Libraries/Mangboard/Board.php +++ /dev/null @@ -1,80 +0,0 @@ -_boards_entity = $boards_entity; - $this->_user_entity = $user_entity; - } - public function getModel(): BoardModel - { - if ($this->_model === null) { - $table_name = "mb_" . $this->_boards_entity->getTitle(); - $this->_model = new BoardModel($table_name); - } - return $this->_model; - } - public function getBoardsEntity(): BoardsEntity - { - return $this->_boards_entity; - } - public function getUserEntity(): UserEntity - { - return $this->_user_entity; - } - - public function createByCrawler(int $cnt, array $listInfo, array $storages): BoardEntity - { - $formDatas = []; - //미디어관련정보 entity에 넣기 - $formDatas['title'] = $listInfo["title"]; - $formDatas['user_pid'] = $this->getUserEntity()->getPK(); - $formDatas['user_id'] = $this->getUserEntity()->getID(); - $formDatas['user_name'] = $listInfo["nickname"] != "" ? $listInfo["nickname"] : $this->getUserEntity()->getTitle(); - $formDatas['level'] = $this->getBoardsEntity()->getListLevel(); - $formDatas['hit'] = $listInfo['hit']; - $formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date'])); - $formDatas['data_type'] = "html"; - $formDatas['editor_type'] = "S"; - $formDatas['image_path'] = ""; - $formDatas['content'] = ""; - foreach ($storages as $storage) { - if ($formDatas['image_path'] == "") { - $formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(); - } - $formDatas['content'] .= $storage->getHTMLTag(); - } - //망보드 게시판에 등록 - if ($formDatas['content'] == "") { - throw new \Exception(sprintf( - "%s=>%s번째 %s 내용이 없어 => %s 등록 취소", - __FUNCTION__, - $cnt, - $listInfo["title"], - $this->getModel()->getTable() - )); - } - $entity = $this->getModel()->create($formDatas); - log_message("notice", sprintf( - "%s=>%s번째 %s => %s 등록 완료", - __FUNCTION__, - $cnt, - $listInfo["title"], - $this->getModel()->getTable() - )); - return $entity; - } -} diff --git a/app/Libraries/Mangboard/Boards.php b/app/Libraries/Mangboard/Boards.php deleted file mode 100644 index 73161b4..0000000 --- a/app/Libraries/Mangboard/Boards.php +++ /dev/null @@ -1,48 +0,0 @@ -_category = $category; - $this->_user_entity = $uer_entity; - } - public function getModel(): BoardsModel - { - if ($this->_model === null) { - $this->_model = new BoardsModel(); - } - return $this->_model; - } - public function getEntity(): BoardsEntity - { - if ($this->_entity === null) { - $this->_entity = $this->getModel()->getEntityByID("board_" . $this->getCategory()); - } - return $this->_entity; - } - //---------------------------------------------------------------------// - public function getCategory(): string - { - return $this->_category; - } - public function getUserEntity(): UserEntity - { - if ($this->_user_entity === null) { - throw new \Exception("사용자정보가 없습니다."); - } - return $this->_user_entity; - } -} diff --git a/app/Libraries/Mangboard/File.php b/app/Libraries/Mangboard/File.php deleted file mode 100644 index 484f603..0000000 --- a/app/Libraries/Mangboard/File.php +++ /dev/null @@ -1,84 +0,0 @@ -_boards_entity = $boards_entity; - $this->_user_entity = $user_entity; - } - public function getModel(): FileModel - { - if ($this->_model === null) { - return $this->_model = new FileModel(); - } - return $this->_model; - } - public function getBoardsEntity(): BoardsEntity - { - return $this->_boards_entity; - } - public function getUserEntity(): UserEntity - { - return $this->_user_entity; - } - public function create(BoardEntity $board_entity, MangboardStorage $storage): FileEntity - { - //파일관리 table에 등록 - $formDatas = []; - //Board PID 넣기 - $formDatas['board_pid'] = $board_entity->getPk(); - $formDatas['user_pid'] = $this->getUserEntity()->getPK(); - $formDatas['user_name'] = $this->getUserEntity()->getTitle(); - $formDatas['board_name'] = $this->getBoardsEntity()->getTitle(); - $formDatas['table_name'] = $this->getModel()->getTable(); - $formDatas['file_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(); - $formDatas['file_name'] = $storage->getOriginName(); - $formDatas['file_type'] = $storage->getMimeType(); - $formDatas['file_caption'] = $storage->getOriginName(); - $formDatas['file_alt'] = $storage->getOriginName(); - $formDatas['file_description'] = "Filedata"; - $formDatas['file_size'] = $storage->getFileSize(); - $formDatas['file_sequence'] = $storage->getOriginSequence(); - $formDatas['reg_date'] = date("Y-m-d H:i:s"); - return $this->getModel()->create($formDatas); - } - public function createByCrawler(BoardEntity $board_entity, array $storages): void - { - try { - foreach ($storages as $storage) { - $entity = $this->create($board_entity, $storage); - log_message("notice", sprintf( - "%s -> %s 게시물의 %s번째:%s 파일 등록 완료", - __FUNCTION__, - $board_entity->getTitle(), - $storage->getOriginSequence(), - $entity->getTitle() - )); - } - } catch (\Exception $e) { - log_message("notice", sprintf( - "\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n", - __FUNCTION__, - $board_entity->getTitle(), - $storage->getOriginSequence(), - $storage->getOriginName(), - $e->getMessage() - )); - } - } -} diff --git a/app/Libraries/Mangboard/Image.php b/app/Libraries/Mangboard/Image.php deleted file mode 100644 index d8dd40f..0000000 --- a/app/Libraries/Mangboard/Image.php +++ /dev/null @@ -1,68 +0,0 @@ -getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL); - $target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target, $fileInfo['extension']); - - if (!$this->isFileType_FileTrait($fileInfo['extension'])) { - throw new \Exception("{$storage->getOriginName()} Image 형식파일이 아닙니다."); - } - // 이미지 파일 로드 - $this->load_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName()); - // 200x200으로 이미지 크기 조정 - $this->resize_ImageTrait($width, $height); - // 파일 저장 - $this->save_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $target_file_name); - // 메모리 해제 - $this->destroy_ImageTrait(); - log_message("debug", sprintf( - "%s %s->%s(W:%s,H:%s) 작업완료)", - __FUNCTION__, - $storage->getOriginName(), - $target_file_name, - $width, - $height - )); - return $target_file_name; - } - public function createByCrawler(BoardEntity $board_entity, array $storages): void - { - try { - foreach ($storages as $storage) { - $file_name = $this->create($storage); - log_message("notice", sprintf( - "%s -> %s 게시물의 %s번째:%s 작은이미지 생성 완료", - __FUNCTION__, - $board_entity->getTitle(), - $storage->getOriginSequence(), - $file_name - )); - } - } catch (\Exception $e) { - log_message("notice", sprintf( - "\n---%s -> %s 게시물의 %s번째:%s 작은이미지 생성 오류---\n%s\n--------------------------------\n", - __FUNCTION__, - $board_entity->getTitle(), - $storage->getOriginSequence(), - $storage->getOriginName(), - $e->getMessage() - )); - } - } -} diff --git a/app/Libraries/Mangboard/User.php b/app/Libraries/Mangboard/User.php deleted file mode 100644 index e31d2f8..0000000 --- a/app/Libraries/Mangboard/User.php +++ /dev/null @@ -1,48 +0,0 @@ -_user_model === null) { - return $this->_user_model = new UserModel(); - } - return $this->_user_model; - } - // 로그인 메서드 - public function login(string $host, string $id, string $password): bool|UserEntity - { - $entity = $this->getUserModel()->getEntityByID($id); - // $response = $this->getWebLibrary($host)->getResponse( - // $host . getenv("mangboard.login.url"), - // "post", - // [ - // 'form_params' => [ - // 'user_id' => $id, - // 'password' => $password, - // ], - // ] - // ); - // if ($response->getStatusCode() == 200) { - // $entity = $this->getUserModel()->getEntityByLoginCheck($id, $password); - // if ($entity === null) { - // throw new \Exception("{$id}는 회원이 아니거나 암호가 맞지 않습니다."); - // } - // } else { - // throw new \Exception("연결실패:" . $response->getStatusCode()); - // } - log_message("notice", "{$id}로 로그인 성공"); - return $entity; - } -} diff --git a/app/Libraries/Cloudflare/Account.php b/app/Libraries/MyCloudflare/Account.php similarity index 100% rename from app/Libraries/Cloudflare/Account.php rename to app/Libraries/MyCloudflare/Account.php diff --git a/app/Libraries/CloudflareLibrary.php b/app/Libraries/MyCloudflare/CloudflareLibrary.php similarity index 100% rename from app/Libraries/CloudflareLibrary.php rename to app/Libraries/MyCloudflare/CloudflareLibrary.php diff --git a/app/Libraries/Cloudflare/Firewall.php b/app/Libraries/MyCloudflare/Firewall.php similarity index 100% rename from app/Libraries/Cloudflare/Firewall.php rename to app/Libraries/MyCloudflare/Firewall.php diff --git a/app/Libraries/Cloudflare/Magictransit/AllowList.php b/app/Libraries/MyCloudflare/Magictransit/AllowList.php similarity index 100% rename from app/Libraries/Cloudflare/Magictransit/AllowList.php rename to app/Libraries/MyCloudflare/Magictransit/AllowList.php diff --git a/app/Libraries/Cloudflare/Record.php b/app/Libraries/MyCloudflare/Record.php similarity index 100% rename from app/Libraries/Cloudflare/Record.php rename to app/Libraries/MyCloudflare/Record.php diff --git a/app/Libraries/Cloudflare/Zone.php b/app/Libraries/MyCloudflare/Zone.php similarity index 100% rename from app/Libraries/Cloudflare/Zone.php rename to app/Libraries/MyCloudflare/Zone.php diff --git a/app/Libraries/MyCrawler/MyCrawler.php b/app/Libraries/MyCrawler/MyCrawler.php index cc2eee3..51088d2 100644 --- a/app/Libraries/MyCrawler/MyCrawler.php +++ b/app/Libraries/MyCrawler/MyCrawler.php @@ -10,16 +10,13 @@ abstract class MyCrawler extends CommonLibrary { use FileTrait; private $_mySocket = null; - protected $_storages = []; protected function __construct($mySocket) { parent::__construct(); $this->_mySocket = $mySocket; } abstract protected function getMyStorage(); - abstract protected function list_page(): array; - abstract protected function detail_page(array $listInfo): array; - abstract protected function backend_process(int $i, array $listInfo, array $storages); + abstract protected function detail_page(int $cnt, array $listInfo): void; final protected function getMySocket() { if ($this->_mySocket === null) { @@ -70,14 +67,14 @@ abstract class MyCrawler extends CommonLibrary } //--------미디어 관련------- - private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): void + private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): mixed { log_message("debug", __FUNCTION__ . " 원본파일 {$file_name} 작업 시작"); $this->getMyStorage()->setOriginName($file_name); $this->getMyStorage()->setOriginContent($content); $this->getMyStorage()->setOriginType($media_type); $this->getMyStorage()->setOriginSequence($file_sequence); - $this->_storages[] = $this->getMyStorage()->save(); + return $this->getMyStorage()->save(); } //Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다. private function media_download(string $media_type, string $url): array @@ -98,19 +95,17 @@ abstract class MyCrawler extends CommonLibrary } final protected function media_process(array $media_urls): array { + log_message("debug", var_export($media_urls, true)); $file_sequence = 1; - $this->_storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 + $storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 // log_message("debug", var_export($urls, true)); foreach ($media_urls as $media_type => $urls) { foreach ($urls as $url) { try { - if ($url === null) { - continue; - } list($file_name, $content) = $this->media_download($media_type, $url); - $this->media_save($file_sequence, $media_type, $file_name, $content); + $storages[] = $this->media_save($file_sequence, $media_type, $file_name, $content); $file_sequence++; - log_message("notice", __FUNCTION__ . " OriginType->{$media_type} 작업 완료"); + log_message("notice", __FUNCTION__ . " MediaType->{$media_type} 작업 완료"); } catch (\Exception $e) { log_message("warning", sprintf( "\n---%s mediaType->%s 오류---\n%s\n-----------------------------------------\n", @@ -121,10 +116,7 @@ abstract class MyCrawler extends CommonLibrary } } } - if (!count($this->_storages)) { - throw new \Exception("Download된 Content가 없습니다."); - } - return $this->_storages; + return $storages; } protected function main_process(int $max_limit, array $listInfos): void { @@ -141,11 +133,7 @@ abstract class MyCrawler extends CommonLibrary try { log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업시작"); //listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_page 처리 안에서 바뀔 수 있으므로 다시 반환 받는다. - list($listInfo, $media_urls) = $this->detail_page($listInfo); - //Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리 - $this->media_process($media_urls); - //File DB 및 Board DB 등록작업등 - $this->backend_process($i, $listInfo, $this->_storages); + $this->detail_page($i, $listInfo); log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업완료."); $i++; } catch (\Exception $e) { diff --git a/app/Libraries/MyCrawler/YamapCrawler.php b/app/Libraries/MyCrawler/YamapCrawler.php index f046e5a..9cad4fb 100644 --- a/app/Libraries/MyCrawler/YamapCrawler.php +++ b/app/Libraries/MyCrawler/YamapCrawler.php @@ -2,13 +2,18 @@ namespace App\Libraries\MyCrawler; +use App\Entities\Mangboard\UserEntity; use App\Libraries\MySocket\WebSocket; use App\Libraries\MyStorage\MangboardStorage; -use App\Entities\Mangboard\UserEntity; +use App\Models\Mangboard\BoardModel; +use App\Models\Mangboard\BoardsModel; +use App\Models\Mangboard\FileModel; +use App\Traits\ImageTrait; use Symfony\Component\DomCrawler\Crawler; class YamapCrawler extends MyCrawler { + use ImageTrait; private $_category = ""; private $_user_entity = null; private $_myStorage = null; @@ -55,60 +60,88 @@ class YamapCrawler extends MyCrawler // //
// - protected function detail_page(array $listInfo): array + protected function detail_page(int $cnt, array $listInfo): void { $response = $this->getMySocket()->getContent($listInfo['detail_url']); $tag = getenv("yamap.view.content.tag"); - return $this->getMediaUrls($response, $tag, $listInfo); - } - protected function list_page(): array - { - if ($this->getDebug()) { - return [ - 'title' => getenv("yamap.view.test.title"), - 'nickname' => getenv("yamap.view.test.nickname"), - 'detail_url' => getenv("yamap.view.test.url"), - 'time' => date("Y-m-d H:i:s"), - 'hit' => 1, - ]; + list($listInfo, $media_urls) = $this->getMediaUrls($response, $tag, $listInfo); + //Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리 + $storages = $this->media_process($media_urls); + if (!count($storages)) { + throw new \Exception("등록할 자료가 없습니다."); } - $listInfos = []; - $response = $this->getMySocket()->getContent(getenv("yamap.list.url.{$this->_category}")); - $selector = $this->getSelector($response, getenv("yamap.list.tag")); - //div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 - // log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); - $selector->filter(getenv("yamap.list.item.tag"))->each( - function (Crawler $node) use (&$listInfos): void { - //bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool - $nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text(); - $hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text(); - $date = $node->filter(getenv("yamap.list.item.date.tag"))->text(); - if ($nickname != getenv("yamap.list.item.nickname.except")) { - //작성자가 "관리자"가 아니 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 - $link_node = $node->filter(getenv("yamap.list.item.link.tag")); - $detail_url = $link_node->attr("href"); - $title = $link_node->children()->last()->text(); - $listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit]; - } - } + //File DB 및 Board DB 등록작업등 + $baord_name = "board_" . $this->_category; + $boardsModel = new BoardsModel(); + $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category); + if ($boards_entity === null) { + throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다."); + } + $boardModel = new BoardModel("mb_" . $baord_name); + $board_entity = $boardModel->createByCrawler( + $boards_entity, + $this->_user_entity, + $cnt, + $listInfo, + $storages ); - if (!count($listInfos)) { - throw new \Exception("Target URL이 없습니다."); + if ($board_entity === null) { + throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다."); } - log_message("notice", __FUNCTION__ . " 작업 완료"); - return $listInfos; - } - protected function backend_process(int $i, array $listInfo, array $storages) - { - //File DB 및 Board DB 등록작업 - $board_entity = $this->getMyStorage()->getBoard()->createByCrawler($i, $listInfo, $storages); - $this->getMyStorage()->getFile()->createByCrawler($board_entity, $storages); - $this->getMyStorage()->getImage()->createByCrawler($board_entity, $storages); + $fileModel = new FileModel(); + $fileModel->createByCrawler( + $boards_entity, + $this->_user_entity, + $board_entity, + $boardModel->getTable(), + $storages + ); + $this->create_small_ImageTrait($board_entity, $storages); } public function execute(int $max_limit): void { - $listInfos = $this->list_page(); - $this->main_process($max_limit, $listInfos); - log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + try { + $listInfos = []; + if ($this->getDebug()) { + $listInfos = [ + 'title' => getenv("yamap.view.test.title"), + 'nickname' => getenv("yamap.view.test.nickname"), + 'detail_url' => getenv("yamap.view.test.url"), + 'time' => date("Y-m-d H:i:s"), + 'hit' => 1, + ]; + } else { + $response = $this->getMySocket()->getContent(getenv("yamap.list.url.{$this->_category}")); + $selector = $this->getSelector($response, getenv("yamap.list.tag")); + //div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 + // log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); + $selector->filter(getenv("yamap.list.item.tag"))->each( + function (Crawler $node) use (&$listInfos): void { + //bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool + $nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text(); + $hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text(); + $date = $node->filter(getenv("yamap.list.item.date.tag"))->text(); + if ($nickname != getenv("yamap.list.item.nickname.except")) { + //작성자가 "관리자"가 아니 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 + $link_node = $node->filter(getenv("yamap.list.item.link.tag")); + $detail_url = $link_node->attr("href"); + $title = $link_node->children()->last()->text(); + $listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit]; + } + } + ); + } + if (!count($listInfos)) { + throw new \Exception("Target URL이 없습니다."); + } + $this->main_process($max_limit, $listInfos); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + } catch (\Exception $e) { + log_message("warning", sprintf( + "\n---%s 오류---\n%s\n-----------------------------------------\n", + __FUNCTION__, + $e->getMessage() + )); + } } } diff --git a/app/Libraries/MyCrawler/YamoonCrawler.php b/app/Libraries/MyCrawler/YamoonCrawler.php index f68462f..9dcde66 100644 --- a/app/Libraries/MyCrawler/YamoonCrawler.php +++ b/app/Libraries/MyCrawler/YamoonCrawler.php @@ -2,13 +2,18 @@ namespace App\Libraries\MyCrawler; +use App\Entities\Mangboard\UserEntity; use App\Libraries\MySocket\WebSocket; use App\Libraries\MyStorage\MangboardStorage; -use App\Entities\Mangboard\UserEntity; +use App\Models\Mangboard\BoardModel; +use App\Models\Mangboard\BoardsModel; +use App\Models\Mangboard\FileModel; +use App\Traits\ImageTrait; use Symfony\Component\DomCrawler\Crawler; class YamoonCrawler extends MyCrawler { + use ImageTrait; private $_category = ""; private $_user_entity = null; private $_myStorage = null; @@ -25,7 +30,8 @@ class YamoonCrawler extends MyCrawler } return $this->_myStorage; } - protected function detail_page(array $listInfo): array + + protected function detail_page(int $cnt, array $listInfo): void { $response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']); //작성시간 @@ -33,60 +39,88 @@ class YamoonCrawler extends MyCrawler // $listInfo['date'] = trim($selector->text()); //작성내용 $tag = getenv("yamoon.view.content.tag"); - return $this->getMediaUrls($response, $tag, $listInfo); - } - protected function list_page(): array - { - if ($this->getDebug()) { - $listInfos = [ - 'title' => getenv("yamoon.view.test.title"), - 'nickname' => getenv("yamoon.view.test.nickname"), - 'detail_url' => getenv("yamoon.view.test.url"), - 'time' => date("Y-m-d H:i:s"), - 'hit' => 1, - ]; - } else { + list($listInfo, $media_urls) = $this->getMediaUrls($response, $tag, $listInfo); + //Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리 + $storages = $this->media_process($media_urls); + if (!count($storages)) { + throw new \Exception("등록할 자료가 없습니다."); } - $listInfos = []; - $response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_category}")); - //div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 - // log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); - // - // - // 졸고 있는 여군 - // 6 - // yeeyuu | 6 | 369 | No 89372 | 2024-09-13 - // - //bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool - $this->getSelector($response, getenv("yamoon.list.tag"))->each( - function (Crawler $node) use (&$listInfos): void { - $link_node = $node->filter(getenv("yamoon.list.item.link.tag")); - $detail_url = $link_node->attr("href"); - $title = $link_node->text(); - $info_node = $node->filter(getenv("yamoon.list.item.info.tag")); - $infos = explode("|", $info_node->text()); - if (trim($infos[4]) == date("Y-m-d")) { - $listInfos[] = ['title' => $title, 'detail_url' => $detail_url, 'nickname' => trim($infos[0]), 'hit' => trim($infos[2]), 'date' => trim($infos[4])]; - } - } + //File DB 및 Board DB 등록작업등 + $baord_name = "board_" . $this->_category; + $boardsModel = new BoardsModel(); + $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category); + if ($boards_entity === null) { + throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다."); + } + $boardModel = new BoardModel("mb_" . $baord_name); + $board_entity = $boardModel->createByCrawler( + $boards_entity, + $this->_user_entity, + $cnt, + $listInfo, + $storages ); - if (!count($listInfos)) { - throw new \Exception("Target URL이 없습니다."); + if ($board_entity === null) { + throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다."); } - log_message("notice", __FUNCTION__ . " 작업 완료"); - return $listInfos; - } - //File DB 및 Board DB 등록작업등 - protected function backend_process(int $i, array $listInfo, array $storages) - { - $board_entity = $this->getMyStorage()->getBoard()->createByCrawler($i, $listInfo, $storages); - $this->getMyStorage()->getFile()->createByCrawler($board_entity, $storages); - $this->getMyStorage()->getImage()->createByCrawler($board_entity, $storages); + $fileModel = new FileModel(); + $fileModel->createByCrawler( + $boards_entity, + $this->_user_entity, + $board_entity, + $boardModel->getTable(), + $storages + ); + $this->create_small_ImageTrait($board_entity, $storages); } + public function execute(int $max_limit): void { - $listInfos = $this->list_page(); - $this->main_process($max_limit, $listInfos); - log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + try { + $listInfos = []; + if ($this->getDebug()) { + $listInfos = [ + 'title' => getenv("yamoon.view.test.title"), + 'nickname' => getenv("yamoon.view.test.nickname"), + 'detail_url' => getenv("yamoon.view.test.url"), + 'time' => date("Y-m-d H:i:s"), + 'hit' => 1, + ]; + } else { + $response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_category}")); + //div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 + // log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); + // + // + // 졸고 있는 여군 + // 6 + // yeeyuu | 6 | 369 | No 89372 | 2024-09-13 + // + //bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool + $this->getSelector($response, getenv("yamoon.list.tag"))->each( + function (Crawler $node) use (&$listInfos): void { + $link_node = $node->filter(getenv("yamoon.list.item.link.tag")); + $detail_url = $link_node->attr("href"); + $title = $link_node->text(); + $info_node = $node->filter(getenv("yamoon.list.item.info.tag")); + $infos = explode("|", $info_node->text()); + if (trim($infos[4]) == date("Y-m-d")) { + $listInfos[] = ['title' => $title, 'detail_url' => $detail_url, 'nickname' => trim($infos[0]), 'hit' => trim($infos[2]), 'date' => trim($infos[4])]; + } + } + ); + } + if (!count($listInfos)) { + throw new \Exception("Target URL이 없습니다."); + } + $this->main_process($max_limit, $listInfos); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + } catch (\Exception $e) { + log_message("warning", sprintf( + "\n---%s 오류---\n%s\n-----------------------------------------\n", + __FUNCTION__, + $e->getMessage() + )); + } } } diff --git a/app/Libraries/MyStorage/MangboardStorage.php b/app/Libraries/MyStorage/MangboardStorage.php index 616641e..1b206ce 100644 --- a/app/Libraries/MyStorage/MangboardStorage.php +++ b/app/Libraries/MyStorage/MangboardStorage.php @@ -82,39 +82,4 @@ class MangboardStorage extends FileStorage )); return $content; } - - private function getBoards(): Boards - { - if ($this->_boards === null) { - $this->_boards = new Boards($this->getCategory(), $this->getUserEntity()); - } - return $this->_boards; - } - final public function getBoard(): Board - { - if ($this->_board === null) { - $this->_board = new Board( - $this->getBoards()->getEntity(), - $this->getUserEntity() - ); - } - return $this->_board; - } - final public function getFile(): File - { - if ($this->_file === null) { - $this->_file = new File( - $this->getBoards()->getEntity(), - $this->getUserEntity() - ); - } - return $this->_file; - } - final public function getImage(): Image - { - if ($this->_image === null) { - $this->_image = new Image(); - } - return $this->_image; - } } diff --git a/app/Models/Mangboard/BoardModel.php b/app/Models/Mangboard/BoardModel.php index 273d631..78f5c4d 100644 --- a/app/Models/Mangboard/BoardModel.php +++ b/app/Models/Mangboard/BoardModel.php @@ -2,8 +2,10 @@ namespace App\Models\Mangboard; -use App\Models\CommonModel; use App\Entities\Mangboard\BoardEntity; +use App\Entities\Mangboard\BoardsEntity; +use App\Entities\Mangboard\UserEntity; +use App\Models\CommonModel; // +-----------------+---------------------+------+-----+---------------------+----------------+ // | Field | Type | Null | Key | Default | Extra | @@ -147,4 +149,52 @@ class BoardModel extends CommonModel { return $this->modify_process($entity, $formDatas); } + public function createByCrawler( + BoardsEntity $boards_entity, + UserEntity $user_entity, + int $cnt, + array $listInfo, + array $storages + ): BoardEntity { + $formDatas = []; + //미디어관련정보 entity에 넣기 + $formDatas['title'] = $listInfo["title"]; + $formDatas['user_pid'] = $user_entity->getPK(); + $formDatas['user_id'] = $user_entity->getID(); + $formDatas['user_name'] = $listInfo["nickname"] != "" ? $listInfo["nickname"] : $user_entity->getTitle(); + $formDatas['level'] = $boards_entity->getListLevel(); + $formDatas['hit'] = $listInfo['hit']; + $formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date'])); + $formDatas['data_type'] = "html"; + $formDatas['editor_type'] = "S"; + $formDatas['image_path'] = ""; + $formDatas['content'] = ""; + foreach ($storages as $storage) { + if ($formDatas['image_path'] == "") { + $formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(); + } + $formDatas['content'] .= $storage->getHTMLTag();; + } + //망보드 게시판에 등록 + if ($formDatas['content'] == "") { + throw new \Exception(sprintf( + "%s=>%s번째 %s 내용이 없어 => %s 등록 안함 : storage->%s", + __FUNCTION__, + $cnt, + $listInfo["title"], + $this->getTable(), + count($storages) + )); + } + $entity = $this->create($formDatas); + log_message("notice", sprintf( + "%s=>%s번째 %s => %s 등록 완료 : storage->%s", + __FUNCTION__, + $cnt, + $listInfo["title"], + $this->getTable(), + count($storages) + )); + return $entity; + } } diff --git a/app/Models/Mangboard/FileModel.php b/app/Models/Mangboard/FileModel.php index 296feb4..4c31a0f 100644 --- a/app/Models/Mangboard/FileModel.php +++ b/app/Models/Mangboard/FileModel.php @@ -2,8 +2,11 @@ namespace App\Models\Mangboard; -use App\Models\CommonModel; +use App\Entities\Mangboard\BoardEntity; +use App\Entities\Mangboard\BoardsEntity; use App\Entities\Mangboard\FileEntity; +use App\Entities\Mangboard\UserEntity; +use App\Models\CommonModel; // +------------------+----------------------+------+-----+---------------------+----------------+ // | Field | Type | Null | Key | Default | Extra | @@ -121,4 +124,50 @@ class FileModel extends CommonModel { return $this->modify_process($entity, $formDatas); } + public function createByCrawler( + BoardsEntity $boards_entity, + UserEntity $user_entity, + BoardEntity $board_entity, + string $board_table, + array $storages + ): void { + foreach ($storages as $storage) { + try { + //파일관리 table에 등록 + $formDatas = []; + //Board PID 넣기 + $formDatas['board_pid'] = $board_entity->getPk(); + $formDatas['user_pid'] = $user_entity->getPK(); + $formDatas['user_name'] = $user_entity->getTitle(); + $formDatas['board_name'] = $boards_entity->getTitle(); + $formDatas['table_name'] = $board_table; + $formDatas['file_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(); + $formDatas['file_name'] = $storage->getOriginName(); + $formDatas['file_type'] = $storage->getMimeType(); + $formDatas['file_caption'] = $storage->getOriginName(); + $formDatas['file_alt'] = $storage->getOriginName(); + $formDatas['file_description'] = "Filedata"; + $formDatas['file_size'] = $storage->getFileSize(); + $formDatas['file_sequence'] = $storage->getOriginSequence(); + $formDatas['reg_date'] = date("Y-m-d H:i:s"); + $entity = $this->create($formDatas); + log_message("notice", sprintf( + "%s -> %s 게시물의 %s번째:%s 파일 등록 완료", + __FUNCTION__, + $board_entity->getTitle(), + $storage->getOriginSequence(), + $entity->getTitle() + )); + } catch (\Exception $e) { + log_message("notice", sprintf( + "\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n", + __FUNCTION__, + $board_entity->getTitle(), + $storage->getOriginSequence(), + $storage->getOriginName(), + $e->getMessage() + )); + } + } + } } diff --git a/app/Traits/ImageTrait.php b/app/Traits/ImageTrait.php index 9ab1853..95bc545 100644 --- a/app/Traits/ImageTrait.php +++ b/app/Traits/ImageTrait.php @@ -2,6 +2,8 @@ namespace App\Traits; +use App\Entities\Mangboard\BoardEntity; + trait ImageTrait { private $_image; @@ -97,4 +99,49 @@ trait ImageTrait { imagedestroy($this->_image); } + + public function create_small_ImageTrait(BoardEntity $board_entity, array $storages, $target_name = "small", int $width = 480, int $height = 319): void + { + try { + foreach ($storages as $storage) { + $fileInfo = pathinfo($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL); + $target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target_name, $fileInfo['extension']); + if (!$this->isFileType_FileTrait($fileInfo['extension'])) { + throw new \Exception("{$storage->getOriginName()} Image 형식파일이 아닙니다."); + } + // 이미지 파일 로드 + $this->load_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName()); + // 200x200으로 이미지 크기 조정 + $this->resize_ImageTrait($width, $height); + // 파일 저장 + $this->save_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $target_file_name); + // 메모리 해제 + $this->destroy_ImageTrait(); + log_message("debug", sprintf( + "%s %s->%s(W:%s,H:%s) 작업완료)", + __FUNCTION__, + $storage->getOriginName(), + $target_file_name, + $width, + $height + )); + log_message("notice", sprintf( + "%s -> %s 게시물의 %s번째:%s 작은이미지 생성 완료", + __FUNCTION__, + $board_entity->getTitle(), + $storage->getOriginSequence(), + $target_file_name + )); + } + } catch (\Exception $e) { + log_message("notice", sprintf( + "\n---%s -> %s 게시물의 %s번째:%s 작은이미지 생성 오류---\n%s\n--------------------------------\n", + __FUNCTION__, + $board_entity->getTitle(), + $storage->getOriginSequence(), + $storage->getOriginName(), + $e->getMessage() + )); + } + } }