From 40e47b281552e41baffe7d8000a77516228047ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=B5=9C=EC=A4=80=ED=9D=A0?= Date: Sun, 8 Sep 2024 03:51:23 +0900 Subject: [PATCH] Automation init...1 --- app/Controllers/CLI/Crawler.php | 41 +--------- app/Libraries/Mangboard/FileLibrary.php | 51 ++++++++---- app/Libraries/MyCrawler/MyCrawlerLibrary.php | 4 +- app/Libraries/MyCrawler/YamapLibrary.php | 83 +++++++++++++++++--- app/Libraries/MyStorage/FileLibrary.php | 8 +- 5 files changed, 118 insertions(+), 69 deletions(-) diff --git a/app/Controllers/CLI/Crawler.php b/app/Controllers/CLI/Crawler.php index 7fc6759..98faebd 100644 --- a/app/Controllers/CLI/Crawler.php +++ b/app/Controllers/CLI/Crawler.php @@ -4,10 +4,6 @@ namespace App\Controllers\CLI; use App\Controllers\BaseController; use App\Libraries\MyCrawler\YamapLibrary; -use App\Libraries\Mangboard\BoardLibrary; -use App\Models\Mangboard\UserModel; -use App\Models\Mangboard\BoardModel; -use App\Entities\Mangboard\BoardEntity; class Crawler extends BaseController { @@ -15,44 +11,9 @@ class Crawler extends BaseController { try { $isDebug = in_array("debug", $params); - //1. 사이트 로그인 처리 - $userModel = new UserModel(); - $user = $userModel->getEntityByID("idcjp"); - // if (!in_array("skip_login", $params)) { - // $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url')); - // $daemonidc->setDebug($isDebug); - // $daemonidc->login( - // getenv('daemonidc.login.url'), - // getenv('daemonidc.login.user_id'), - // getenv('daemonidc.login.user_password') - // ); - // } - //2.Yamap사이트에서 자유게시판에서 최근 게시물 데이터 가져오기 $yamap = new YamapLibrary(); $yamap->setDebug($isDebug); - list($item, $fileInfos) = $yamap->execute(); - - //3.망보드 일반게시판에 게시물 등록 처리 - if (!in_array("skip_create", $params)) { - $board = new BoardLibrary(new BoardModel(getenv("crawler.yamap.registration.table"))); - $board->setDebug($isDebug); - - //미디어관련정보 entity에 넣기 - $entity = new BoardEntity(); - $entity->title = $item["title"]; - $entity->user_name = $item["nickname"]; - $entity->reg_date = $item['time']; - $entity->hit = $item['hit']; - $entity->data_type = "html"; - $entity->editor_type = "S"; - foreach ($fileInfos as $fileInfo) { - $entity->content .= $fileInfo["mediatag"]; - } - - //망보드에 넣기 - $board->create($entity); - } - + $yamap->execute(); log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다."); return true; } catch (\Exception $e) { diff --git a/app/Libraries/Mangboard/FileLibrary.php b/app/Libraries/Mangboard/FileLibrary.php index a03896b..778a4c2 100644 --- a/app/Libraries/Mangboard/FileLibrary.php +++ b/app/Libraries/Mangboard/FileLibrary.php @@ -11,8 +11,10 @@ use App\Entities\Mangboard\FileEntity; class FileLibrary extends MyStorageLibrary { private $_user = null; - private $_boardName = null; + private $_boardName = ""; + private $_boardTable = ""; private $_model = null; + private $_fileEntity = null; public function __construct(string $path) { parent::__construct($path); @@ -39,6 +41,17 @@ class FileLibrary extends MyStorageLibrary { $this->_boardName = $boardName; } + public function getBoardTable(): string + { + if ($this->_boardTable === null) { + throw new \Exception("BoardModel이 지정되지 않았습니다."); + } + return $this->_boardTable; + } + public function setBoardTable(string $boardTable): void + { + $this->_boardName = $boardTable; + } public function getModel(): FileModel { @@ -48,22 +61,32 @@ class FileLibrary extends MyStorageLibrary return $this->_model; } - //망보드 파일관리 table에 등록 - public function createFile($content): bool + public function getFileEntity(): null|FileEntity + { + return $this->_fileEntity; + } + + public function save($content): bool { if (!parent::save($content)) { return false; } - //mb_files 모델작업 - $entity = new FileEntity(); - $entity->setTitle($this->getFileName()); - $entity->user_pid = $this->getUser()->getPK(); - $entity->user_name = $this->getUser()->getTitle(); - $entity->board_name = $this->getBoardName(); - $entity->table_name = "mb_" . $this->getBoardName(); - $entity->file_path = $this->getPath(); - $entity->file_type = $this->getMimeType(); - $entity->reg_date = date("Y-m-d H:i:s"); - return true; + //망보드 파일관리 table에 등록 + try { + //mb_files 모델작업 + $entity = new FileEntity(); + $entity->setTitle($this->getFileName()); + $entity->user_pid = $this->getUser()->getPK(); + $entity->user_name = $this->getUser()->getTitle(); + $entity->board_name = $this->getBoardName(); + $entity->table_name = $this->getBoardTable(); + $entity->file_path = $this->getPath(); + $entity->file_type = $this->getMimeType(); + $entity->reg_date = date("Y-m-d H:i:s"); + $this->_fileEntity = $this->getModel()->create($entity); + return true; + } catch (\Exception $e) { + return false; + } } } diff --git a/app/Libraries/MyCrawler/MyCrawlerLibrary.php b/app/Libraries/MyCrawler/MyCrawlerLibrary.php index afa4b79..4d85e44 100644 --- a/app/Libraries/MyCrawler/MyCrawlerLibrary.php +++ b/app/Libraries/MyCrawler/MyCrawlerLibrary.php @@ -13,7 +13,7 @@ abstract class MyCrawlerLibrary extends CommonLibrary } abstract protected function getMySocket(); abstract protected function getMyStorage(); - abstract public function execute(): array; + abstract public function execute(): void; final protected function getContent(string $url, string $tag): Crawler { @@ -75,7 +75,7 @@ abstract class MyCrawlerLibrary extends CommonLibrary } return $downloadInfos; } - final protected function save(string $mediaType, array $downloadInfos, $fileInfos = []): array + protected function save(string $mediaType, array $downloadInfos, $fileInfos = []): array { foreach ($downloadInfos as $downloadInfo) { $this->getMyStorage()->setFileName($downloadInfo['fileName']); diff --git a/app/Libraries/MyCrawler/YamapLibrary.php b/app/Libraries/MyCrawler/YamapLibrary.php index 42cb1f3..ba30e45 100644 --- a/app/Libraries/MyCrawler/YamapLibrary.php +++ b/app/Libraries/MyCrawler/YamapLibrary.php @@ -2,34 +2,58 @@ namespace App\Libraries\MyCrawler; -use App\Libraries\Mangboard\FileLibrary as MyStorage; -use App\Libraries\MySocket\WebLibrary as MySocket; + use Symfony\Component\DomCrawler\Crawler; +use App\Libraries\MySocket\WebLibrary as MySocket; +use App\Libraries\Mangboard\FileLibrary as MyStorage; +use App\Libraries\Mangboard\BoardLibrary; +use App\Models\Mangboard\BoardModel; +use App\Entities\Mangboard\BoardEntity; +use App\Models\Mangboard\UserModel; class YamapLibrary extends MyCrawlerLibrary { private $_mySocket = null; private $_myStorage = null; + private $_boardModel = null; public function __construct() { parent::__construct(); } - protected function getMySocket() + final public function getMySocket(): mixed { if ($this->_mySocket === null) { $this->_mySocket = new MySocket(getenv('yamap.host.url')); } return $this->_mySocket; } + final public function setMySocket($mySocket): void + { + $this->_mySocket = $mySocket; + } - protected function getMyStorage() + final public function getMyStorage(): mixed { if ($this->_myStorage === null) { $this->_myStorage = new MyStorage(getenv('yamap.storage.upload.path')); + $this->_myStorage->setBoardName(getenv('crawler.yamap.registration.board')); + $this->_myStorage->setBoardTable($this->getBoardModel()->getTable()); } return $this->_myStorage; } + final public function setMyStorage($myStorage): void + { + $this->_myStorage = $myStorage; + } + + private function getBoardModel(): BoardModel + { + if ($this->_boardModel === null) { + $this->_boardModel = new BoardModel(getenv("crawler.yamap.registration.table")); + } + return $this->_boardModel; + } private function mainPage(string $url): array { @@ -57,7 +81,7 @@ class YamapLibrary extends MyCrawlerLibrary return $items; } - private function detailPage(string $url): array + private function detailPage(string $url, array $fileInfos = []): array { $crawler = $this->getContent($url, getenv("yamap.view.content.tag")); //3. Image 처리 @@ -65,13 +89,44 @@ class YamapLibrary extends MyCrawlerLibrary $fileInfos = $this->save("image", $downloadInfos); //4. Video(mp4) 처리 $downloadInfos = $this->download($crawler, ["tag" => "video", "attr" => "src"]); - return $this->save("video", $downloadInfos); + return $this->save("video", $downloadInfos, $fileInfos); } - public function execute(): array + private function createBoard(array $item, array $fileInfos): BoardEntity { + $board = new BoardLibrary($this->getBoardModel()); + $board->setDebug($this->getDebug()); + + //미디어관련정보 entity에 넣기 + $entity = new BoardEntity(); + $entity->title = $item["title"]; + $entity->user_name = $item["nickname"]; + $entity->reg_date = $item['time']; + $entity->hit = $item['hit']; + $entity->data_type = "html"; + $entity->editor_type = "S"; + foreach ($fileInfos as $fileInfo) { + $entity->content .= $fileInfo["mediatag"]; + } + //망보드에 넣기 + return $board->create($entity); + } + + public function execute(): void + { + //1. 사이트 로그인 처리 + $userModel = new UserModel(); + $user = $userModel->getEntityByID("idcjp"); + // $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url')); + // $daemonidc->setDebug($isDebug); + // $daemonidc->login( + // getenv('daemonidc.login.url'), + // getenv('daemonidc.login.user_id'), + // getenv('daemonidc.login.user_password') + // ); + $this->getMyStorage()->setUser($user); $items = []; - //1. 해당사이트 MainPage 처리 + //2. 해당사이트 MainPage 처리 if ($this->getDebug()) { $items[] = [ 'title' => getenv("yamap.view.test.title"), @@ -83,8 +138,14 @@ class YamapLibrary extends MyCrawlerLibrary } else { $items = $this->mainPage(getenv("yamap.list.url")); } - //2. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리 - $fileInfos = $this->detailPage($items[0]["detail_url"]); - return array($items[0], $fileInfos); + if (!count($items)) { + throw new \Exception("Yamap 사이트에서 게시물이 존재하지 않습니다."); + } + //최초 게시물만 등록하기 위함 + $item = $items[0]; + //3. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리 + $fileInfos = $this->detailPage($item["detail_url"]); + //4.망보드 일반게시판에 게시물 등록 처리 + $this->createBoard($item, $fileInfos); } } diff --git a/app/Libraries/MyStorage/FileLibrary.php b/app/Libraries/MyStorage/FileLibrary.php index e86d26b..c275cb2 100644 --- a/app/Libraries/MyStorage/FileLibrary.php +++ b/app/Libraries/MyStorage/FileLibrary.php @@ -40,8 +40,12 @@ class FileLibrary extends MyStorageLibrary } } $saveFile = $fullPath . DIRECTORY_SEPARATOR . $this->getFileName(); - $this->_mimeType = mime_content_type($saveFile); log_message("debug", "Storage Save-> " . $saveFile); - return file_put_contents($saveFile, $content); + if (!file_put_contents($saveFile, $content)) { + return false; + } + //File형식에 따른 MimeType 지정 + $this->_mimeType = mime_content_type($saveFile); + return true; } }