Automation init...3

This commit is contained in:
최준흠 2024-09-12 00:28:10 +09:00
parent dfb5808584
commit c40399b7ca
10 changed files with 230 additions and 282 deletions

View File

@ -21,7 +21,10 @@ $routes->group('admin', ['namespace' => 'App\Controllers\Admin', 'filter' => 'au
$routes->get('', 'UserController::index');
});
});
$routes->group('yamap', ['namespace' => 'App\Controllers\Yamap'], function ($routes) {});
$routes->group('crawler', ['namespace' => 'App\Controllers'], function ($routes) {
$routes->cli('yamap', 'CrawlerController::yamap');
$routes->cli('yamap/(:any)', 'CrawlerController::yamap/$1');
});
$routes->group('mangboard', ['namespace' => 'App\Controllers\Mangboard'], function ($routes) {
$routes->group('user', function ($routes) {
$routes->get('/', 'UserController::index');
@ -31,10 +34,6 @@ $routes->group('mangboard', ['namespace' => 'App\Controllers\Mangboard'], functi
$routes->cli('check_level', 'UserController::check_level');
$routes->cli('check_level/(:alpha)', 'UserController::check_level/$1');
});
$routes->group('crawler', function ($routes) {
$routes->cli('yamap', 'CrawlerController::yamap');
$routes->cli('yamap/(:any)', 'CrawlerController::yamap/$1');
});
$routes->group('image', function ($routes) {
$routes->cli('yamap', 'ImageController::yamap');
$routes->cli('yamap/(:any)', 'ImageController::yamap/$1');

View File

@ -0,0 +1,44 @@
<?php
namespace App\Controllers;
use App\Libraries\MySocket\WebLibrary as MySocketLibrary;
use App\Libraries\MyStorage\MangboardFileLibrary as MyStorageLibrary;
use App\Libraries\MyCrawler\YamapLibrary as MyCrawler;
use App\Controllers\CommonController;
use App\Models\Mangboard\UserModel;
class CrawlerController extends CommonController
{
public function yamap(...$params): string
{
try {
//1. 사이트 로그인 처리
$userModel = new UserModel();
$userEntity = $userModel->getEntityByID("idcjp");
// $daemonidc = new MySocketLibrary(getenv('daemonidc.host.url'));
// $daemonidc->setDebug($isDebug);
// $daemonidc->login(
// getenv('daemonidc.login.url'),
// getenv('daemonidc.login.user_id'),
// getenv('daemonidc.login.user_password')
// );
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$mySocket = new MySocketLibrary(getenv('yamap.host.url'));
$myStorage = new MyStorageLibrary(getenv('yamap.storage.upload.path'));
$myStorage->setUser($userEntity);
$myStorage->setBoardName(getenv('yamap.storage.board.name'));
$myStorage->setBoardlevel(getenv('yamap.storage.board.level'));
$crawler = new MyCrawler($mySocket, $myStorage);
if (in_array("debug", $params)) {
$crawler->setDebug(true);
}
//3. 실행
$crawler->execute();
return "완료되었습니다.";
} catch (\Exception $e) {
log_message("error", $e->getMessage());
return $e->getMessage();
}
}
}

View File

@ -1,63 +0,0 @@
<?php
namespace App\Controllers\Mangboard;
use App\Controllers\CommonController;
use App\Libraries\MyCrawler\YamapLibrary as MyCrawler;
class CrawlerController extends CommonController
{
public function yamap(...$params): string
{
try {
$crawler = new MyCrawler();
if (in_array("debug", $params)) {
$crawler->setDebug(true);
}
//1. 사이트 로그인 처리
$user = $crawler->login();
$crawler->getMyStorage()->setUser($user);
$itemInfos = [];
//2. 해당사이트 MainPage 처리
if ($crawler->getDebug()) {
$itemInfos[] = [
'title' => getenv("yamap.view.test.title"),
'nickname' => getenv("yamap.view.test.nickname"),
'detail_url' => getenv("yamap.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1
];
} else {
$itemInfos = $crawler->mainPage(getenv("yamap.list.url"));
}
if (!count($itemInfos)) {
throw new \Exception("Yamap 사이트에서 게시물이 존재하지 않습니다.");
}
//Limit가 0이면 $itemInfos 갯수만큼 다하고, LIMIT 갯수 혹은 $items의 갯수중 작은수만큼 한다.
$max_limit = intval(getenv("yamap.list.max_limit"));
if ($max_limit) {
$max_limit = count($itemInfos) <= $max_limit ? count($itemInfos) : $max_limit;
} else {
$max_limit = count($itemInfos);
}
$i = 1;
foreach ($itemInfos as $itemInfo) {
if ($i < $max_limit) {
try {
log_message("notice", "게시물 {$i}번째 {$itemInfo["nickname"]} 작업시작");
$crawler->detailPage($itemInfo);
log_message("notice", "게시물 {$i}번째 {$itemInfo["nickname"]} 작업완료.");
$i++;
} catch (\Exception $e) {
log_message("debug", $e->getMessage());
}
}
}
log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다.");
return "완료되었습니다.";
} catch (\Exception $e) {
log_message("error", $e->getMessage());
return $e->getMessage();
}
}
}

View File

@ -2,18 +2,58 @@
namespace App\Entities\Mangboard;
use App\Entities\MyStorage\FileEntity as MyStorageEntity;
use App\Entities\CommonEntity;
class FileEntity extends MyStorageEntity
class FileEntity extends CommonEntity
{
public function __toString(): string
{
return "{$this->getPK()}|" . parent::__toString();
return "{$this->getPK()}|{$this->getPath()}|{$this->getTitle()}|{$this->getMimeType()}}|{$this->getSequence()}번째";
}
public function getTitle(): string
{
return $this->attributes['file_name'];
}
public function setTitle(string $file_name): void
{
$this->attributes['file_name'] = $file_name;
}
//Common Function
public function getPK(): int
{
return $this->attributes['pid'];
}
public function getPath(): string
{
return $this->attributes['file_path'];
}
public function setPath(string $file_path): void
{
$this->attributes['file_path'] = $file_path;
}
final public function getMimeType(): string
{
return $this->attributes['file_type'];
}
public function setMimeType(string $mimetype): void
{
$this->attributes['file_type'] = $mimetype;
}
final public function getSize(): int
{
return $this->attributes['file_size'];
}
public function setSize(int $file_size): void
{
$this->attributes['file_size'] = $file_size;
}
//한게시물에 여러개가 있을경우 번호
final public function getSequence(): int
{
return $this->attributes['file_sequence'];
}
public function setSequence(int $file_sequence): void
{
$this->attributes['file_sequence'] = $file_sequence;
}
}

View File

@ -1,68 +0,0 @@
<?php
namespace App\Entities\MyStorage;
use App\Entities\CommonEntity;
class FileEntity extends CommonEntity
{
public function __construct(array|null $data = null)
{
parent::__construct($data);
}
public function __toString(): string
{
return "{$this->getPath()}|{$this->getTitle()}|{$this->getMimeType()}}|{$this->getSequence()}번째";
}
public function getTitle(): string
{
return $this->attributes['file_name'];
}
public function setTitle(string $file_name): void
{
$this->attributes['file_name'] = $file_name;
}
//Common Function
public function getPath(): string
{
return $this->attributes['file_path'];
}
public function setPath(string $file_path): void
{
$this->attributes['file_path'] = $file_path;
}
final public function getMimeType(): string
{
return $this->attributes['file_type'];
}
public function setMimeType(string $mimetype): void
{
$this->attributes['file_type'] = $mimetype;
}
final public function getSize(): int
{
return $this->attributes['file_size'];
}
public function setSize(int $file_size): void
{
$this->attributes['file_size'] = $file_size;
}
final public function getMediaHTML(): string
{
return $this->attributes['media_html'];
}
public function setMediaHTML(string $media_html): void
{
$this->attributes['media_html'] = $media_html;
}
//한게시물에 여러개가 있을경우 번호
final public function getSequence(): int
{
return $this->attributes['file_sequence'];
}
public function setSequence(int $file_sequence): void
{
$this->attributes['file_sequence'] = $file_sequence;
}
}

View File

@ -2,17 +2,35 @@
namespace App\Libraries\MyCrawler;
use App\Libraries\CommonLibrary;
use Symfony\Component\DomCrawler\Crawler;
use App\Libraries\CommonLibrary;
abstract class MyCrawlerLibrary extends CommonLibrary
{
protected function __construct()
private $_mySocket = null;
private $_myStorage = null;
protected function __construct($mySocket, $myStorage)
{
parent::__construct();
$this->_mySocket = $mySocket;
$this->_myStorage = $myStorage;
}
abstract public function execute(): void;
final protected function getMySocket(): mixed
{
if ($this->_mySocket === null) {
throw new \Exception("MySocket이 지정되지 않았습니다.");
}
return $this->_mySocket;
}
final protected function getMyStorage(): mixed
{
if ($this->_myStorage === null) {
throw new \Exception("MySocket이 지정되지 않았습니다.");
}
return $this->_myStorage;
}
abstract public function getMySocket();
abstract public function getMyStorage();
final protected function getContent(string $url, string $tag): Crawler
{
@ -24,22 +42,16 @@ abstract class MyCrawlerLibrary extends CommonLibrary
return $crawler->filter($tag);
}
private function getNodes(Crawler $crawler, array $options, $nodes = []): array
//Download한 파일 저장후 추가작업시 사용
final protected function download(string $mediaType, Crawler $crawler, array $options, array $myStorageLibrarys = []): array
{
$nodes = [];
$crawler->filter($options["tag"])->each(
function (Crawler $node) use (&$options, &$nodes): void {
log_message("debug", sprintf("getNode->%s[%s]", $options["tag"], $node->attr($options['attr'])));
$nodes[] = $node;
}
);
return $nodes;
}
//Download한 파일 저장후 추가작업시 사용
final protected function download(string $mediaType, Crawler $crawler, array $options, array $results = []): array
{
$nodes = $this->getNodes($crawler, $options);
$file_sequence = 1;
foreach ($nodes as $node) {
try {

View File

@ -2,29 +2,18 @@
namespace App\Libraries\MyCrawler;
use App\Libraries\MySocket\WebLibrary as MySocketLibrary;
use App\Libraries\MyStorage\Mangboard\FileLibrary as MyStorageLibrary;
use App\Models\Mangboard\UserModel;
use App\Models\Mangboard\BoardModel;
use App\Entities\Mangboard\UserEntity;
use Symfony\Component\DomCrawler\Crawler;
class YamapLibrary extends MyCrawlerLibrary
{
private $_userModel = null;
private $_boardModel = null;
private $_mySocket = null;
private $_myStorage = null;
public function __construct()
public function __construct($mySocket, $myStorage)
{
parent::__construct();
}
private function getUserModel(): UserModel
{
if ($this->_userModel === null) {
$this->_userModel = new UserModel();
}
return $this->_userModel;
parent::__construct($mySocket, $myStorage);
//원래는 mb_board Table에서 해당Board정보를 읽어서 처리해아함
$this->getMyStorage()->setBoardTable($this->getBoardModel()->getTable());
}
private function getBoardModel(): BoardModel
{
@ -33,41 +22,38 @@ class YamapLibrary extends MyCrawlerLibrary
}
return $this->_boardModel;
}
public function getMySocket(): MySocketLibrary
private function createBoard(array $itemInfo, array $myStorageLibrarys): void
{
if ($this->_mySocket === null) {
$this->_mySocket = new MySocketLibrary(getenv('yamap.host.url'));
//미디어관련정보 entity에 넣기
$formDatas = [];
$formDatas['title'] = $itemInfo["title"];
$formDatas['user_pid'] = $this->getMyStorage()->getUser()->getPK();
$formDatas['user_id'] = $this->getMyStorage()->getUser()->getID();
$formDatas['user_name'] = $itemInfo["nickname"] != "" ? $itemInfo["nickname"] : $this->getMyStorage()->getUser()->getTitle();
$formDatas['level'] = $this->getMyStorage()->getBoardLevel();
$formDatas['hit'] = $itemInfo['hit'];
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($itemInfo['date']));
$formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S";
$formDatas['image_path'] = false;
$content = "";
foreach ($myStorageLibrarys as $myStorageLibrary) {
if ($formDatas['image_path'] === false) {
$formDatas['image_path'] = $myStorageLibrary->getPath();
}
$content .= $myStorageLibrary->getMediaTag();
}
return $this->_mySocket;
}
public function getMyStorage(): MyStorageLibrary
{
if ($this->_myStorage === null) {
$this->_myStorage = new MyStorageLibrary(getenv('yamap.storage.upload.path'));
//원래는 mb_board에서 해당Board정보를 읽어서 처리해아함
$this->_myStorage->setBoardName(getenv('yamap.storage.board.name'));
$this->_myStorage->setBoardTable($this->getBoardModel()->getTable());
$this->_myStorage->setBoardlevel(getenv('yamap.storage.board.level'));
$formDatas['content'] = $content;
//망보드 게시판에 등록
$entity = $this->getBoardModel()->create($formDatas);
//망보드 파일관리툴에 등록된 파일게시물에 등록한 게시판번호 수정하기
foreach ($myStorageLibrarys as $myStorageLibrary) {
$myStorageLibrary->setBoardPID(intval($entity->getPK()));
}
return $this->_myStorage;
log_message("notice", __FUNCTION__ . " 작업 완료");
}
public function login(): UserEntity
{
$entity = $this->getUserModel()->getEntityByID("idcjp");
// $daemonidc = new MySocketLibrary(getenv('daemonidc.host.url'));
// $daemonidc->setDebug($isDebug);
// $daemonidc->login(
// getenv('daemonidc.login.url'),
// getenv('daemonidc.login.user_id'),
// getenv('daemonidc.login.user_password')
// );
return $entity;
}
public function mainPage(string $url): array
private function mainPage(string $url): array
{
$crawler = $this->getContent($url, getenv("yamap.list.tag"));
$items = [];
@ -94,46 +80,58 @@ class YamapLibrary extends MyCrawlerLibrary
return $items;
}
private function createBoard(array $itemInfo, array $myStorageLibrarys): void
{
//미디어관련정보 entity에 넣기
$formDatas = [];
$formDatas['title'] = $itemInfo["title"];
$formDatas['user_pid'] = $this->getMyStorage()->getUser()->getPK();
$formDatas['user_id'] = $this->getMyStorage()->getUser()->getID();
$formDatas['user_name'] = $itemInfo["nickname"] != "" ? $itemInfo["nickname"] : $this->getMyStorage()->getUser()->getTitle();
$formDatas['level'] = $this->getMyStorage()->getBoardLevel();
$formDatas['hit'] = $itemInfo['hit'];
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($itemInfo['date']));
$formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S";
$formDatas['image_path'] = false;
foreach ($myStorageLibrarys as $myStorageLibrary) {
if ($formDatas['image_path'] === false) {
$formDatas['image_path'] = $myStorageLibrary->getFileEntity()->getPath();
}
$formDatas['content'] .= $myStorageLibrary->getFileEntity()->getMediaHTML();
}
//망보드 게시판에 등록
$entity = $this->getBoardModel()->create($formDatas);
//망보드 파일관리툴에 등록된 파일게시물에 등록한 게시판번호 수정하기
foreach ($myStorageLibrarys as $myStorageLibrary) {
$myStorageLibrary->setBoardPID(intval($entity->getPK()));
}
log_message("notice", __FUNCTION__ . " 작업 완료");
}
public function detailPage(array $itemInfo)
private function detailPage(array $itemInfo)
{
$crawler = $this->getContent($itemInfo['detail_url'], getenv("yamap.view.content.tag"));
//3. Image 처리
$myStorageLibrarys = $this->download("image", $crawler, ["tag" => "img", "attr" => "src"]);
//4. Video(mp4) 처리
$myStorageLibrarys = $this->download("video", $crawler, ["tag" => "video", "attr" => "src"], $myStorageLibrarys);
log_message("notice", __FUNCTION__ . " 작업 완료");
//5.망보드 일반게시판에 게시물 등록 처리
if (count($myStorageLibrarys)) {
$this->createBoard($itemInfo, $myStorageLibrarys);
}
log_message("notice", __FUNCTION__ . " 작업 완료");
}
public function execute(): void
{
//. 해당사이트 MainPage 처리
$itemInfos = [];
if ($this->getDebug()) {
$itemInfos[] = [
'title' => getenv("yamap.view.test.title"),
'nickname' => getenv("yamap.view.test.nickname"),
'detail_url' => getenv("yamap.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1
];
} else {
$itemInfos = $this->mainPage(getenv("yamap.list.url"));
}
if (!count($itemInfos)) {
throw new \Exception("Yamap 사이트에서 게시물이 존재하지 않습니다.");
}
//Limit가 0이면 $itemInfos 갯수만큼 다하고, LIMIT 갯수 혹은 $items의 갯수중 작은수만큼 한다.
$max_limit = intval(getenv("yamap.list.max_limit"));
if ($max_limit) {
$max_limit = count($itemInfos) <= $max_limit ? count($itemInfos) : $max_limit;
} else {
$max_limit = count($itemInfos);
}
$i = 1;
foreach ($itemInfos as $itemInfo) {
if ($i < $max_limit) {
try {
log_message("notice", "게시물 {$i}번째 {$itemInfo["nickname"]} 작업시작");
$this->detailPage($itemInfo);
log_message("notice", "게시물 {$i}번째 {$itemInfo["nickname"]} 작업완료.");
$i++;
} catch (\Exception $e) {
log_message("debug", $e->getMessage());
}
}
}
log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다.");
}
}

View File

@ -2,12 +2,12 @@
namespace App\Libraries\MyStorage;
use App\Entities\MyStorage\FileEntity;
use App\Traits\FileTrait;
class FileLibrary extends MyStorageLibrary
{
use FileTrait;
private $_uploadPath = "uploads";
private $_path = "";
private $_mimeType = "";
private $_fileSize = 0;
@ -18,6 +18,10 @@ class FileLibrary extends MyStorageLibrary
parent::__construct();
$this->_path = $path;
}
final public function getUploadPath(): string
{
return $this->_uploadPath;
}
final public function getPath(): string
{
return $this->_path;
@ -34,10 +38,10 @@ class FileLibrary extends MyStorageLibrary
{
return $this->_fileSequence;
}
protected function getMediaTag(string $mediaType): string
final public function getMediaTag(): string
{
$mediaTag = "";
switch ($mediaType) {
switch ($this->getOrintginType()) {
case "image":
$mediaTag = sprintf(
"<img src=\"/%s/%s/%s\" alt=\"%s\">",

View File

@ -1,6 +1,6 @@
<?php
namespace App\Libraries\MyStorage\Mangboard;
namespace App\Libraries\MyStorage;
use App\Libraries\MyStorage\FileLibrary as MyStorageLibrary;
use App\Libraries\MyUtil\ImageLibrary;
@ -8,7 +8,7 @@ use App\Models\Mangboard\FileModel;
use App\Entities\Mangboard\UserEntity;
use App\Entities\Mangboard\FileEntity;
class FileLibrary extends MyStorageLibrary
class MangboardFileLibrary extends MyStorageLibrary
{
private $_user = null;
private $_boardName = "";
@ -89,7 +89,7 @@ class FileLibrary extends MyStorageLibrary
}
//작은이미지 생성
private function save_small_image(int $width = 480, int $height = 319): bool
private function save_smallimage(int $width = 480, int $height = 319): string
{
$fullPath = WRITEPATH . $this->getUploadPath() . DIRECTORY_SEPARATOR . $this->getPath();
$image = new ImageLibrary();
@ -107,18 +107,41 @@ class FileLibrary extends MyStorageLibrary
$this->getOriginSequence(),
$this->getPath()
));
return $result;
//정석방식
// if ($result) {
// if ($result) {
// //작은이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
// return = sprintf(
// "%s/%s",
// $entity->getPath(),
// $image->getDestinationFile()
// ));
// } else {
// //원본이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
// return sprintf(
// "%s/%s",
// $entity->getPath(),
// $this->getOriginName()
// ));
// }
// }
//망보드 방식
//mb_files에서 file_path가 망보드 게시판 파일관리에서 image로 표시되어 file_path+file_name로 설정
//원본이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
return sprintf("%s/%s", $this->getPath(), $this->getOriginName());
}
//망보드 파일관리 table에 등록
private function save_db(array $formDatas): FileEntity
public function save(): static
{
parent::save();
$formDatas = [];
//작은이미지생성후 Path/파일명 넣기
$formDatas['file_path'] = $this->save_smallimage();
$formDatas['user_pid'] = $this->getUser()->getPK();
$formDatas['user_name'] = $this->getUser()->getTitle();
$formDatas['board_name'] = $this->getBoardName();
$formDatas['table_name'] = $this->getBoardTable();
$formDatas['file_name'] = $this->getOriginName();
$formDatas['file_path'] = $this->getOriginName();
$formDatas['file_type'] = $this->getMimeType();
$formDatas['file_caption'] = $this->getOriginName();
$formDatas['file_alt'] = $this->getOriginName();
@ -126,49 +149,14 @@ class FileLibrary extends MyStorageLibrary
$formDatas['file_size'] = $this->getFileSize();
$formDatas['file_sequence'] = $this->getOriginSequence();
$formDatas['reg_date'] = date("Y-m-d H:i:s");
// log_message("debug", "\n-----Entity Value-----\n" . var_export($entity->toArray(), true) . "\n---------------------------\n");
//망보드 파일관리 table에 등록
$entity = $this->getModel()->create($formDatas);
log_message("notice", sprintf(
"%s %s번째 작업 완료",
__FUNCTION__,
$this->getOriginSequence()
));
return $entity;
}
public function save(): static
{
parent::save();
$formDatas = [];;
if ($this->save_small_image()) {
//개인적인 생각방식
// if ($result) {
// //작은이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
// $entity->setPath(sprintf(
// "%s/%s",
// $entity->getPath(),
// $image->getDestinationFile()
// ));
// } else {
// //원본이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
// $entity->setPath(sprintf(
// "%s/%s",
// $entity->getPath(),
// $this->getOriginName()
// ));
// }
//개인적인 생각방식
//
//망보드 방식
//mb_files에서 file_path가 망보드 게시판 파일관리에서 image로 표시되어 file_path+file_name로 설정
//원본이미지 생성후 목적지 Path와 파일명을 다시 file_path에 넣는다. URL이 되기때문에 /로 넣어야함
$formDatas['file_path'] = sprintf(
"%s/%s",
$this->getPath(),
$this->getOriginName()
);
//망보드 방식
}
$this->setFileEntity($this->save_db($formDatas));
$this->setFileEntity($entity);
return $this;
}
}

View File

@ -6,7 +6,6 @@ use App\Libraries\CommonLibrary;
abstract class MyStorageLibrary extends CommonLibrary
{
private $_uploadPath = "uploads";
private $_originName = "";
private $_originContent = "";
private $_originType = "";
@ -15,12 +14,7 @@ abstract class MyStorageLibrary extends CommonLibrary
{
parent::__construct();
}
abstract public function save(): mixed;
final public function getUploadPath(): string
{
return $this->_uploadPath;
}
final public function getOriginName(): string
{
return $this->_originName;