From 46ee8d605da005a3eb6bf09c58d6140392afe89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=B5=9C=EC=A4=80=ED=9D=A0?= Date: Tue, 10 Sep 2024 18:04:49 +0900 Subject: [PATCH] Automation init...3 --- app/Config/Routes.php | 6 +- app/Controllers/Crawler/YamapController.php | 58 ++++++-- .../{Auth => MyAuth}/GoogleLibrary.php | 5 +- .../{Auth => MyAuth}/LocalLibrary.php | 5 +- .../MyAuthLibrary.php} | 4 +- app/Libraries/MyCrawler/MyCrawlerLibrary.php | 3 +- app/Libraries/MyCrawler/YamapLibrary.php | 4 +- app/Libraries/MyStorage/FileLibrary.php | 16 +-- app/Libraries/MyStorage/MangboardLibrary.php | 40 ++++-- app/Libraries/MyUtil/ImageLibrary.php | 124 ++++++++++++++++++ app/Libraries/MyUtil/MyUtilLibrary.php | 13 ++ app/Models/CommonModel.php | 2 +- app/Models/Mangboard/FileModel.php | 16 +++ app/Traits/FileTrait.php | 42 ++++++ 14 files changed, 293 insertions(+), 45 deletions(-) rename app/Libraries/{Auth => MyAuth}/GoogleLibrary.php (96%) rename app/Libraries/{Auth => MyAuth}/LocalLibrary.php (87%) rename app/Libraries/{Auth/AuthLibrary.php => MyAuth/MyAuthLibrary.php} (95%) create mode 100644 app/Libraries/MyUtil/ImageLibrary.php create mode 100644 app/Libraries/MyUtil/MyUtilLibrary.php create mode 100644 app/Traits/FileTrait.php diff --git a/app/Config/Routes.php b/app/Config/Routes.php index 266e97c..d951df3 100644 --- a/app/Config/Routes.php +++ b/app/Config/Routes.php @@ -16,8 +16,10 @@ $routes->addPlaceholder('uuid', '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4} $routes->get('/', 'Home::index'); $routes->group('crawler', ['namespace' => 'App\Controllers\Crawler'], function ($routes) { - $routes->cli('yamap', 'YamapController::crawling'); - $routes->cli('yamap/debug', 'YamapController::crawling/debug'); + $routes->cli('yamap/crawling', 'YamapController::crawling'); + $routes->cli('yamap/crawling/(:any)', 'YamapController::crawling/$1'); + $routes->cli('yamap/small_image', 'YamapController::small_image'); + $routes->cli('yamap/small_image/(:any)', 'YamapController::small_image/$1'); }); $routes->group('admin', ['namespace' => 'App\Controllers\Admin', 'filter' => 'authFilter:manager'], function ($routes) { diff --git a/app/Controllers/Crawler/YamapController.php b/app/Controllers/Crawler/YamapController.php index 675fed5..63132f0 100644 --- a/app/Controllers/Crawler/YamapController.php +++ b/app/Controllers/Crawler/YamapController.php @@ -4,17 +4,40 @@ namespace App\Controllers\Crawler; use App\Controllers\CommonController; use App\Libraries\MyCrawler\YamapLibrary as MyCrawler; +use App\Libraries\MyUtil\ImageLibrary; class YamapController extends CommonController { - public function crawling(string $option = ""): string + public function small_image(...$params) + { + try { + $fullPath = WRITEPATH . "uploads" . DIRECTORY_SEPARATOR . getenv('yamap.storage.upload.path'); + $image = new ImageLibrary( + $fullPath, + $fullPath . DIRECTORY_SEPARATOR . "small" + ); + if (in_array("debug", $params)) { + $image->setDebug(true); + } + //디렉토리 생성 + $image->makeDirectory($image->getDestinationPath()); + foreach ($image->getFilesByExtentionType($image->getSourcePath()) as $file) { + $image->make_small_image($file); + } + log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다."); + return "완료되었습니다."; + } catch (\Exception $e) { + log_message("error", $e->getMessage()); + return $e->getMessage(); + } + } + public function crawling(...$params): string { try { $crawler = new MyCrawler(); - if ($option === "debug") { + if (in_array("debug", $params)) { $crawler->setDebug(true); } - //1. 사이트 로그인 처리 $user = $crawler->login(); $crawler->getMyStorage()->setUser($user); @@ -34,19 +57,28 @@ class YamapController extends CommonController if (!count($items)) { throw new \Exception("Yamap 사이트에서 게시물이 존재하지 않습니다."); } + //Limit가 0이면 $items 갯수만큼 다하고, LIMIT 갯수 혹은 $items의 갯수중 작은수만큼 한다. + $max_limit = intval(getenv("yamap.list.max_limit")); + if ($max_limit) { + $max_limit = count($items) <= $max_limit ? count($items) : $max_limit; + } else { + $max_limit = count($items); + } $i = 0; foreach ($items as $item) { - try { - //3. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리 - $fileEntitys = $crawler->detailPage($item["detail_url"]); - //4.망보드 일반게시판에 게시물 등록 처리 - if (count($fileEntitys)) { - $crawler->createBoard($item, $fileEntitys); + if ($i <= $max_limit) { + try { + //3. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리 + $fileEntitys = $crawler->detailPage($item["detail_url"]); + //4.망보드 일반게시판에 게시물 등록 처리 + if (count($fileEntitys)) { + $crawler->createBoard($item, $fileEntitys); + } + $i++; + log_message("notice", "게시물 {$i}번째 {$item["nickname"]} 작업완료."); + } catch (\Exception $e) { + log_message("debug", $e->getMessage()); } - $i++; - log_message("notice", "게시물 {$i}번째 {$item["nickname"]} 작업완료."); - } catch (\Exception $e) { - log_message("debug", $e->getMessage()); } } log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다."); diff --git a/app/Libraries/Auth/GoogleLibrary.php b/app/Libraries/MyAuth/GoogleLibrary.php similarity index 96% rename from app/Libraries/Auth/GoogleLibrary.php rename to app/Libraries/MyAuth/GoogleLibrary.php index 5609a11..71c3822 100644 --- a/app/Libraries/Auth/GoogleLibrary.php +++ b/app/Libraries/MyAuth/GoogleLibrary.php @@ -1,13 +1,12 @@ getMySocket()->download($node->attr($options["attr"])); $entitys[] = $this->getMyStorage()->save($fileName, $mediaType, $content, $file_sequence); $file_sequence++; + log_message("notice", __FUNCTION__ . " {$mediaType} 작업 완료"); } catch (\Exception $e) { - log_message("notice", $e->getMessage()); + log_message("warning", "---" . __FUNCTION__ . "--.\n" . $e->getMessage()) . "-----------------------------------------\n"; } } return $entitys; diff --git a/app/Libraries/MyCrawler/YamapLibrary.php b/app/Libraries/MyCrawler/YamapLibrary.php index 28769b3..8f4e465 100644 --- a/app/Libraries/MyCrawler/YamapLibrary.php +++ b/app/Libraries/MyCrawler/YamapLibrary.php @@ -119,7 +119,7 @@ class YamapLibrary extends MyCrawlerLibrary $entity->image_path = false; foreach ($fileEntitys as $fileEntity) { if ($entity->image_path === false) { - $entity->image_path = $fileEntity->getPath() . DIRECTORY_SEPARATOR . $fileEntity->getTitle(); + $entity->image_path = $fileEntity->getPath(); } $entity->content .= $fileEntity->getMediaHTML(); } @@ -133,7 +133,7 @@ class YamapLibrary extends MyCrawlerLibrary $entity = $this->getBoardModel()->create($entity); //망보드 파일관리툴에 등록된 파일게시물에 등록한 게시판번호 수정하기 $this->getMyStorage()->setBoardPID($fileEntitys, $entity->getPK()); - log_message("debug", $this->getBoardModel()->getTable() . " Table에 등록 완료"); + log_message("notice", __FUNCTION__ . " 작업 완료"); return $entity; } } diff --git a/app/Libraries/MyStorage/FileLibrary.php b/app/Libraries/MyStorage/FileLibrary.php index d11597b..fd45fff 100644 --- a/app/Libraries/MyStorage/FileLibrary.php +++ b/app/Libraries/MyStorage/FileLibrary.php @@ -3,9 +3,11 @@ namespace App\Libraries\MyStorage; use App\Entities\MyStorage\FileEntity; +use App\Traits\FileTrait; class FileLibrary extends MyStorageLibrary { + use FileTrait; private $_path = ""; public function __construct(string $path) { @@ -55,26 +57,22 @@ class FileLibrary extends MyStorageLibrary public function save(string $fileName, string $mediaType, string $content, int $file_sequence): FileEntity { $fullPath = WRITEPATH . $this->getUploadPath() . DIRECTORY_SEPARATOR . $this->getPath(); - if (!is_dir($fullPath)) { - if (!mkdir($fullPath)) { - throw new \Exception("디렉토리 생성 실패:{$fullPath}"); - } - } + $this->makeDirectory($fullPath); $saveFilePath = $fullPath . DIRECTORY_SEPARATOR . $fileName; if (file_exists($saveFilePath)) { - throw new \Exception("이미 존재하는 파일:{$saveFilePath}"); + throw new \Exception(__FUNCTION__ . "이미 존재하는 파일:{$saveFilePath}"); } if (!file_put_contents($saveFilePath, $content)) { - throw new \Exception("파일저장 실패:{$saveFilePath}"); + throw new \Exception(__FUNCTION__ . "파일저장 실패:{$saveFilePath}"); } $entity = $this->getFileEntity(); $entity->setPath($this->getPath()); $entity->setTitle($fileName); $entity->setMimeType(mime_content_type($saveFilePath)); - $entity->setSize(!filesize($saveFilePath) ?: 0); + $entity->setSize(filesize($saveFilePath)); $entity->setSequence($file_sequence); $entity->setMediaHTML($this->getMediaTag($mediaType, $entity)); - log_message("debug", "{$file_sequence}번째 " . $entity->getTitle() . " 파일저장 완료"); + log_message("notice", __FUNCTION__ . " {$file_sequence}번째 " . $entity->getTitle() . " 작업 완료"); return $entity; } } diff --git a/app/Libraries/MyStorage/MangboardLibrary.php b/app/Libraries/MyStorage/MangboardLibrary.php index a5e279f..58af7e2 100644 --- a/app/Libraries/MyStorage/MangboardLibrary.php +++ b/app/Libraries/MyStorage/MangboardLibrary.php @@ -3,12 +3,13 @@ namespace App\Libraries\MyStorage; -use App\Libraries\MyStorage\FileLibrary as MyStorageLibrary; -use App\Models\Mangboard\FileModel; -use App\Entities\Mangboard\UserEntity; use App\Entities\Mangboard\FileEntity; +use App\Entities\Mangboard\UserEntity; +use App\Libraries\MyStorage\FileLibrary; +use App\Libraries\MyUtil\ImageLibrary; +use App\Models\Mangboard\FileModel; -class MangboardLibrary extends MyStorageLibrary +class MangboardLibrary extends FileLibrary { private $_user = null; private $_boardName = ""; @@ -74,21 +75,41 @@ class MangboardLibrary extends MyStorageLibrary { $this->_boardLevel = $boardLevel; } - public function save(string $fileName, string $mediaType, string $content, int $file_sequence): FileEntity + private function create_small_image(FileEntity $entity, int $file_sequence): FileEntity + { + $fullPath = WRITEPATH . $this->getUploadPath() . DIRECTORY_SEPARATOR . $this->getPath(); + $image = new ImageLibrary( + $fullPath, + $fullPath . DIRECTORY_SEPARATOR . "small" + ); + $image->setDebug($this->getDebug()); + //Small 디렉토리 생성 + $image->makeDirectory($image->getDestinationPath()); + $dstfile = $image->make_small_image($entity->getTitle()); + log_message("notice", __FUNCTION__ . " {$file_sequence}번째 {$dstfile} 작업 완료"); + return $entity; + } + //망보드 파일관리 table에 등록 + private function create_db(FileEntity $entity, int $file_sequence): FileEntity { - $entity = parent::save($fileName, $mediaType, $content, $file_sequence); // log_message("debug", $this->getModel()->getTable() . " Table에 {$file_sequence}번째 등록 준비->{$entity->getTitle()}|{$entity->getMimeType()}"); - //망보드 파일관리 table에 등록 $entity->user_pid = $this->getUser()->getPK(); $entity->user_name = $this->getUser()->getTitle(); $entity->board_name = $this->getBoardName(); $entity->table_name = $this->getBoardTable(); $entity->reg_date = date("Y-m-d H:i:s"); - $entity->file_caption = $fileName; + $entity->file_caption = $entity->getTitle(); $entity->file_alt = $entity->getTitle(); $entity->file_description = "Filedata"; $entity = $this->getModel()->create($entity); - log_message("debug", $this->getModel()->getTable() . " Table에 {$file_sequence}번째 등록 완료->{$entity}"); + log_message("notice", __FUNCTION__ . " {$file_sequence}번째 작업 완료"); + return $entity; + } + public function save(string $fileName, string $mediaType, string $content, int $file_sequence): FileEntity + { + $entity = parent::save($fileName, $mediaType, $content, $file_sequence); + $entity = $this->create_small_image($entity, $file_sequence); + $entity = $this->create_db($entity, $file_sequence); return $entity; } public function setBoardPID(array $fileEntitys, int $board_pid) @@ -98,5 +119,6 @@ class MangboardLibrary extends MyStorageLibrary $fileEntity->board_pid = $board_pid; $this->getModel()->modify($fileEntity); } + log_message("notice", __FUNCTION__ . " 작업 완료"); } } diff --git a/app/Libraries/MyUtil/ImageLibrary.php b/app/Libraries/MyUtil/ImageLibrary.php new file mode 100644 index 0000000..3149ebb --- /dev/null +++ b/app/Libraries/MyUtil/ImageLibrary.php @@ -0,0 +1,124 @@ +_srcPath = $srcPath; + $this->_dstPath = $dstPath; + } + // 이미지의 현재 너비를 반환하는 메소드 + public function getSourcePath(): string + { + return $this->_srcPath; + } + public function getDestinationPath(): string + { + return $this->_dstPath; + } + public function getWidth() + { + return imagesx($this->_image); + } + // 이미지의 현재 높이를 반환하는 메소드 + public function getHeight() + { + return imagesy($this->_image); + } + // 이미지 파일을 로드하는 메소드 + private function load($file) + { + $imageInfo = getimagesize($file); + $this->_imageType = $imageInfo[2]; + switch ($this->_imageType) { + case IMAGETYPE_JPEG: + $this->_image = imagecreatefromjpeg($file); + break; + case IMAGETYPE_GIF: + $this->_image = imagecreatefromgif($file); + break; + case IMAGETYPE_PNG: + $this->_image = imagecreatefrompng($file); + break; + case IMAGETYPE_WEBP: + $this->_image = imagecreatefromwebp($file); + break; + } + } + // 이미지 크기를 지정된 너비, 높이로 변경하는 메소드 + private function resize($width, $height) + { + $newImage = imagecreatetruecolor($width, $height); + imagecopyresampled($newImage, $this->_image, 0, 0, 0, 0, $width, $height, $this->getWidth(), $this->getHeight()); + $this->_image = $newImage; + } + // 이미지 비율을 유지하면서 크기를 조정하는 메소드 + private function resizeToWidth($width) + { + $ratio = $width / $this->getWidth(); + $height = $this->getHeight() * $ratio; + $this->resize($width, $height); + } + private function resizeToHeight($height) + { + $ratio = $height / $this->getHeight(); + $width = $this->getWidth() * $ratio; + $this->resize($width, $height); + } + private function scale($scale) + { + $width = $this->getWidth() * ($scale / 100); + $height = $this->getHeight() * ($scale / 100); + $this->resize($width, $height); + } + // 이미지를 저장하는 메소드 + private function save($file, $imageType = IMAGETYPE_WEBP, $compression = 75) + { + switch ($imageType) { + case IMAGETYPE_JPEG: + imagejpeg($this->_image, $file, $compression); + break; + case IMAGETYPE_GIF: + imagegif($this->_image, $file); + break; + case IMAGETYPE_PNG: + imagepng($this->_image, $file); + break; + case IMAGETYPE_WEBP: + default: + imagewebp($this->_image, $file, $compression); + break; + } + } + // 메모리 해제를 위한 메소드 + private function destroy() + { + imagedestroy($this->_image); + } + + public function make_small_image(string $file, int $width = 480, int $height = 319): string + { + //소스파일 + $srcfile = $this->getSourcePath() . DIRECTORY_SEPARATOR . $file; + $fileInfos = pathinfo($srcfile, PATHINFO_ALL); + //저장파일 + $dstfile = $this->getDestinationPath() . DIRECTORY_SEPARATOR . $fileInfos['filename'] . "_small." . $fileInfos['extension']; + $this->load($srcfile); // 이미지 파일 로드 + $this->resize($width, $height); // 200x200으로 이미지 크기 조정 + $this->save($dstfile); // 저장 + $this->destroy(); // 메모리 해제 + log_message("notice", __FUNCTION__ . " 작업 완료"); + return $dstfile; + } +} diff --git a/app/Libraries/MyUtil/MyUtilLibrary.php b/app/Libraries/MyUtil/MyUtilLibrary.php new file mode 100644 index 0000000..b3c8bd6 --- /dev/null +++ b/app/Libraries/MyUtil/MyUtilLibrary.php @@ -0,0 +1,13 @@ +$field === null) { return $entity; diff --git a/app/Models/Mangboard/FileModel.php b/app/Models/Mangboard/FileModel.php index 1568070..1dae5b7 100644 --- a/app/Models/Mangboard/FileModel.php +++ b/app/Models/Mangboard/FileModel.php @@ -102,6 +102,22 @@ class FileModel extends CommonModel return $this->getEntity(); } + final protected function convertEntityData($entity, $field): mixed + { + if ($entity->$field === null) { + return $entity; + } + switch ($field) { + case "file_path": + $entity->$field = $entity->$field . DIRECTORY_SEPARATOR . $entity->getTitle(); + break; + default: + $entity = parent::convertEntityData($entity, $field); + break; + } + return $entity; + } + //create용 public function create(FileEntity $entity): FileEntity { diff --git a/app/Traits/FileTrait.php b/app/Traits/FileTrait.php new file mode 100644 index 0000000..5bd407c --- /dev/null +++ b/app/Traits/FileTrait.php @@ -0,0 +1,42 @@ +