Automation init...2

This commit is contained in:
최준흠 2024-09-07 23:25:59 +09:00
parent f61113074f
commit 84245e4c0b
13 changed files with 120 additions and 86 deletions

View File

@ -2,13 +2,12 @@
namespace App\Controllers\CLI; namespace App\Controllers\CLI;
use App\Models\Mangboard\BoardModel; use App\Controllers\BaseController;
use App\Libraries\MyWebLibrary;
use App\Libraries\MyCrawler\YamapLibrary; use App\Libraries\MyCrawler\YamapLibrary;
use App\Libraries\Mangboard\BoardLibrary; use App\Libraries\Mangboard\BoardLibrary;
use App\Models\Mangboard\UserModel;
use App\Models\Mangboard\BoardModel;
use App\Entities\Mangboard\BoardEntity; use App\Entities\Mangboard\BoardEntity;
use App\Controllers\BaseController;
class Crawler extends BaseController class Crawler extends BaseController
{ {
@ -16,13 +15,9 @@ class Crawler extends BaseController
{ {
try { try {
$isDebug = in_array("debug", $params); $isDebug = in_array("debug", $params);
//1.Yamap사이트에서 자유게시판에서 최근 게시물 데이터 가져오기 //1. 사이트 로그인 처리
$yamap = new YamapLibrary(); $userModel = new UserModel();
$yamap->setDebug($isDebug); $user = $userModel->getEntityByID("idcjp");
if (!in_array("skip_build", $params)) {
list($item, $fileInfos, $mediaTags) = $yamap->execute();
}
// //2. 사이트 로그인 처리
// if (!in_array("skip_login", $params)) { // if (!in_array("skip_login", $params)) {
// $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url')); // $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url'));
// $daemonidc->setDebug($isDebug); // $daemonidc->setDebug($isDebug);
@ -32,7 +27,12 @@ class Crawler extends BaseController
// getenv('daemonidc.login.user_password') // getenv('daemonidc.login.user_password')
// ); // );
// } // }
//3. 망보드 일반게시판에 게시물 등록 처리 //2.Yamap사이트에서 자유게시판에서 최근 게시물 데이터 가져오기
$yamap = new YamapLibrary();
$yamap->setDebug($isDebug);
list($item, $fileInfos) = $yamap->execute();
//3.망보드 일반게시판에 게시물 등록 처리
if (!in_array("skip_create", $params)) { if (!in_array("skip_create", $params)) {
$board = new BoardLibrary(new BoardModel(getenv("crawler.yamap.registration.table"))); $board = new BoardLibrary(new BoardModel(getenv("crawler.yamap.registration.table")));
$board->setDebug($isDebug); $board->setDebug($isDebug);
@ -45,11 +45,14 @@ class Crawler extends BaseController
$entity->hit = $item['hit']; $entity->hit = $item['hit'];
$entity->data_type = "html"; $entity->data_type = "html";
$entity->editor_type = "S"; $entity->editor_type = "S";
$entity->content = is_array($mediaTags) ? implode("\n", $mediaTags) : $mediaTags; foreach ($fileInfos as $fileInfo) {
$entity->content .= $fileInfo["mediatag"];
}
//망보드에 넣기 //망보드에 넣기
$board->create($entity); $board->create($entity);
} }
log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다."); log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다.");
return true; return true;
} catch (\Exception $e) { } catch (\Exception $e) {

View File

@ -2,7 +2,6 @@
namespace App\Controllers\CLI; namespace App\Controllers\CLI;
use App\Models\Mangboard\UserModel;
use App\Libraries\Mangboard\UserLibrary; use App\Libraries\Mangboard\UserLibrary;
use App\Controllers\BaseController; use App\Controllers\BaseController;
@ -12,7 +11,7 @@ class Mangboard extends BaseController
{ {
try { try {
$isDebug = in_array("debug", $params); $isDebug = in_array("debug", $params);
$user = new UserLibrary(new UserModel); $user = new UserLibrary();
$user->setDebug($isDebug); $user->setDebug($isDebug);
$user->setLevel(); $user->setLevel();
log_message("notice", "Mangboard->level 작업이 완료되었습니다."); log_message("notice", "Mangboard->level 작업이 완료되었습니다.");

View File

@ -2,7 +2,6 @@
namespace App\Controllers\Mangboard\Admin; namespace App\Controllers\Mangboard\Admin;
use App\Models\Mangboard\UserModel;
use App\Libraries\Mangboard\UserLibrary; use App\Libraries\Mangboard\UserLibrary;
use App\Controllers\Admin\AdminController; use App\Controllers\Admin\AdminController;
@ -25,7 +24,7 @@ class UserController extends AdminController
$point = intval($this->request->getVar("point")); $point = intval($this->request->getVar("point"));
$sign = $this->request->getVar("point") ?? "+"; $sign = $this->request->getVar("point") ?? "+";
$user = new UserLibrary(new UserModel()); $user = new UserLibrary();
$user->setPoint($id, $point, $sign); $user->setPoint($id, $point, $sign);
return "완료되었습니다."; return "완료되었습니다.";
} catch (\Exception $e) { } catch (\Exception $e) {

View File

@ -13,4 +13,5 @@ abstract class CommonEntity extends Entity
abstract public function __toString(); abstract public function __toString();
abstract public function getPK(); abstract public function getPK();
abstract public function getTitle(); abstract public function getTitle();
abstract public function setTitle(string $tile): void;
} }

View File

@ -18,5 +18,9 @@ class BoardEntity extends CommonEntity
{ {
return $this->attributes['title']; return $this->attributes['title'];
} }
public function setTitle(string $title): void
{
$this->attributes['title'] = $title;
}
//Common Function //Common Function
} }

View File

@ -18,5 +18,9 @@ class FileEntity extends CommonEntity
{ {
return $this->attributes['file_name']; return $this->attributes['file_name'];
} }
public function setTitle(string $title): void
{
$this->attributes['file_name'] = $title;
}
//Common Function //Common Function
} }

View File

@ -18,6 +18,10 @@ class UserEntity extends CommonEntity
{ {
return $this->attributes['user_name']; return $this->attributes['user_name'];
} }
public function setTitle(string $title): void
{
$this->attributes['user_name'] = $title;
}
//Common Function //Common Function
public function getID(): string public function getID(): string

View File

@ -15,9 +15,4 @@ abstract class CommonLibrary
{ {
$this->_debug = $debug; $this->_debug = $debug;
} }
final public function getFileMimeType($file): string
{
return image_type_to_mime_type(exif_imagetype($file));
}
} }

View File

@ -3,16 +3,42 @@
namespace App\Libraries\Mangboard; namespace App\Libraries\Mangboard;
use App\Models\Mangboard\FileModel;
use App\Libraries\MyStorage\FileLibrary as MyStorageLibrary; use App\Libraries\MyStorage\FileLibrary as MyStorageLibrary;
use App\Models\Mangboard\FileModel;
use App\Entities\Mangboard\UserEntity;
use App\Entities\Mangboard\FileEntity;
class FileLibrary extends MyStorageLibrary class FileLibrary extends MyStorageLibrary
{ {
private $_user = null;
private $_boardName = null;
private $_model = null; private $_model = null;
public function __construct(string $path) public function __construct(string $path)
{ {
parent::__construct($path); parent::__construct($path);
} }
public function getUser(): UserEntity
{
if ($this->_user === null) {
throw new \Exception("사용자정보가 없습니다.");
}
return $this->_user;
}
public function setUser(UserEntity $user): void
{
$this->_user = $user;
}
public function getBoardName(): string
{
if ($this->_boardName === null) {
throw new \Exception("BoardModel이 지정되지 않았습니다.");
}
return $this->_boardName;
}
public function setBoardName(string $boardName): void
{
$this->_boardName = $boardName;
}
public function getModel(): FileModel public function getModel(): FileModel
{ {
@ -21,13 +47,23 @@ class FileLibrary extends MyStorageLibrary
} }
return $this->_model; return $this->_model;
} }
public function save($content): bool
//망보드 파일관리 table에 등록
public function createFile($content): bool
{ {
if (!parent::save($content)) { if (!parent::save($content)) {
return false; return false;
} }
//mb_files 모델작업 //mb_files 모델작업
$entity = new FileEntity();
$entity->setTitle($this->getFileName());
$entity->user_pid = $this->getUser()->getPK();
$entity->user_name = $this->getUser()->getTitle();
$entity->board_name = $this->getBoardName();
$entity->table_name = "mb_" . $this->getBoardName();
$entity->file_path = $this->getPath();
$entity->file_type = $this->getMimeType();
$entity->reg_date = date("Y-m-d H:i:s");
return true; return true;
} }
} }

View File

@ -11,13 +11,13 @@ abstract class MyCrawlerLibrary extends CommonLibrary
{ {
parent::__construct(); parent::__construct();
} }
abstract public function getMySocket(); abstract protected function getMySocket();
abstract public function getMyStorage(); abstract protected function getMyStorage();
abstract public function execute(): array; abstract public function execute(): array;
final protected function getContent(string $url, string $tag): Crawler final protected function getContent(string $url, string $tag): Crawler
{ {
$response = $this->getMySocket()->getContent($url); $response = $this->getMySocket()->getContent($url);
if (!$response) { if (!$response) {
throw new \Exception("getCrawler 실패:{$url}"); throw new \Exception("getCrawler 실패:{$url}");
} }
@ -35,18 +35,47 @@ abstract class MyCrawlerLibrary extends CommonLibrary
); );
return $nodes; return $nodes;
} }
protected function getMediaTag(string $mediaType): string
{
$mediaTag = "";
switch ($mediaType) {
case "image":
$mediaTag = sprintf(
"<img src=\"/%s/%s/%s\" alt=\"%s\">",
$this->getMyStorage()->getUploadPath(),
$this->getMyStorage()->getPath(),
$this->getMyStorage()->getFieName(),
$this->getMyStorage()->getFieName()
);
break;
case "video":
$mediaTag = sprintf(
"<video alt=\"%s\" controls autoplay>
<source src=\"/%s/%s/%s\" type=\"%s\">
Your browser does not support the video tag.
</video>",
$this->getMyStorage()->getFieName(),
$this->getMyStorage()->getUploadPath(),
$this->getMyStorage()->getPath(),
$this->getMyStorage()->getFieName(),
$this->getMyStorage()->getFieMimeType(),
);
break;
}
return $mediaTag;
}
final protected function download(Crawler $crawler, array $options): array final protected function download(Crawler $crawler, array $options): array
{ {
$downloadInfos = []; $downloadInfos = [];
$nodes = $this->getNodes($crawler, $options); $nodes = $this->getNodes($crawler, $options);
foreach ($nodes as $node) { foreach ($nodes as $node) {
$downloadInfos[] = $this->getMySocket()->download($node->attr($options["attr"])); $url = $node->attr($options["attr"]);
$downloadInfos[] = $this->getMySocket()->download($url);
} }
return $downloadInfos; return $downloadInfos;
} }
final protected function save(string $mediaType, array $downloadInfos, $fileInfos = []): array
final protected function save(array $downloadInfos, $fileInfos = []): array
{ {
foreach ($downloadInfos as $downloadInfo) { foreach ($downloadInfos as $downloadInfo) {
$this->getMyStorage()->setFileName($downloadInfo['fileName']); $this->getMyStorage()->setFileName($downloadInfo['fileName']);
@ -54,47 +83,12 @@ abstract class MyCrawlerLibrary extends CommonLibrary
continue; continue;
} }
$fileInfos[] = [ $fileInfos[] = [
"url" => $downloadInfo['url'], "mediatag" => $this->getMediaTag($mediaType),
"path" => $this->getMyStorage()->getPath(), "path" => $this->getMyStorage()->getPath(),
"fileType" => $this->getMyStorage()->getFieType(), "mimeType" => $this->getMyStorage()->getMimeType(),
"fileName" => $this->getMyStorage()->getFieName(), "fileName" => $this->getMyStorage()->getFieName(),
]; ];
} }
return $fileInfos; return $fileInfos;
} }
final protected function getMediaTags(array $fileInfos, array $mediaTags = []): array
{
switch ($fileInfos['fileType']) {
case "jpeg":
if ($this->getMySocket()->isContainsHttpOrHttps($fileInfos['orignal'])) {
$mediaTags[] = $fileInfos['orignal'];
} else {
$mediaTags[] = sprintf(
"<img src=\"/%s/%s/%s\" alt=\"%s\">",
$this->getMyStorage()->getUploadPath(),
$fileInfos["path"],
$fileInfos["fileName"],
$fileInfos["fileName"]
);
}
break;
case "mp4":
if ($this->getMySocket()->isContainsHttpOrHttps($fileInfos['orignal'])) {
$mediaTags[] = $fileInfos['orignal'];
} else {
$mediaTags[] = sprintf(
"<video src=\"/%s/%s/%s\" alt=\"%s\">",
$this->getMyStorage()->getUploadPath(),
$fileInfos["path"],
$fileInfos["fileName"],
$fileInfos["fileName"]
);
}
break;
default:
$mediaTags[] = $fileInfos['orignal'];
break;
}
return $mediaTags;
}
} }

View File

@ -2,9 +2,9 @@
namespace App\Libraries\MyCrawler; namespace App\Libraries\MyCrawler;
use Symfony\Component\DomCrawler\Crawler;
use App\Libraries\MySocket\WebLibrary as MySocket;
use App\Libraries\Mangboard\FileLibrary as MyStorage; use App\Libraries\Mangboard\FileLibrary as MyStorage;
use App\Libraries\MySocket\WebLibrary as MySocket;
use Symfony\Component\DomCrawler\Crawler;
class YamapLibrary extends MyCrawlerLibrary class YamapLibrary extends MyCrawlerLibrary
{ {
@ -15,7 +15,7 @@ class YamapLibrary extends MyCrawlerLibrary
parent::__construct(); parent::__construct();
} }
public function getMySocket() protected function getMySocket()
{ {
if ($this->_mySocket === null) { if ($this->_mySocket === null) {
$this->_mySocket = new MySocket(getenv('yamap.host.url')); $this->_mySocket = new MySocket(getenv('yamap.host.url'));
@ -23,7 +23,7 @@ class YamapLibrary extends MyCrawlerLibrary
return $this->_mySocket; return $this->_mySocket;
} }
public function getMyStorage() protected function getMyStorage()
{ {
if ($this->_myStorage === null) { if ($this->_myStorage === null) {
$this->_myStorage = new MyStorage(getenv('yamap.storage.upload.path')); $this->_myStorage = new MyStorage(getenv('yamap.storage.upload.path'));
@ -62,15 +62,10 @@ class YamapLibrary extends MyCrawlerLibrary
$crawler = $this->getContent($url, getenv("yamap.view.content.tag")); $crawler = $this->getContent($url, getenv("yamap.view.content.tag"));
//3. Image 처리 //3. Image 처리
$downloadInfos = $this->download($crawler, ["tag" => "img", "attr" => "src"]); $downloadInfos = $this->download($crawler, ["tag" => "img", "attr" => "src"]);
$fileInfos = $this->save($downloadInfos); $fileInfos = $this->save("image", $downloadInfos);
$mediaTags = $this->getMediaTags($fileInfos);
//4. Video(mp4) 처리 //4. Video(mp4) 처리
$downloadInfos = $this->download($crawler, ["tag" => "video", "attr" => "src"]); $downloadInfos = $this->download($crawler, ["tag" => "video", "attr" => "src"]);
$fileInfos = $this->save($downloadInfos); return $this->save("video", $downloadInfos);
$mediaTags = $this->getMediaTags($fileInfos);
log_message("debug", "-----mediaTags-----");
log_message("debug", var_export($mediaTags, true));
return array($fileInfos, $mediaTags);
} }
public function execute(): array public function execute(): array
@ -89,7 +84,7 @@ class YamapLibrary extends MyCrawlerLibrary
$items = $this->mainPage(getenv("yamap.list.url")); $items = $this->mainPage(getenv("yamap.list.url"));
} }
//2. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리 //2. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리
list($fileInfos, $mediaTags) = $this->detailPage($items[0]["detail_url"]); $fileInfos = $this->detailPage($items[0]["detail_url"]);
return array($items[0], $fileInfos, $mediaTags); return array($items[0], $fileInfos);
} }
} }

View File

@ -97,6 +97,6 @@ class WebLibrary extends MySocketLibrary
} }
log_message("notice", "{$fileName} 파일이 다운로드되었습니다!"); log_message("notice", "{$fileName} 파일이 다운로드되었습니다!");
} }
return ["url" => $url, "fileName" => $fileName, "content" => $content]; return ["fileName" => $fileName, "content" => $content];
} }
} }

View File

@ -6,7 +6,7 @@ class FileLibrary extends MyStorageLibrary
{ {
private $_path = ""; private $_path = "";
private $_fileName = ""; private $_fileName = "";
private $_fileType = ""; private $_mimeType = "";
public function __construct(string $path) public function __construct(string $path)
{ {
parent::__construct(); parent::__construct();
@ -26,9 +26,9 @@ class FileLibrary extends MyStorageLibrary
$this->_fileName = $fileName; $this->_fileName = $fileName;
} }
final public function getFileType(): string final public function getMimeType(): string
{ {
return $this->_fileType; return $this->_mimeType;
} }
public function save($content): bool public function save($content): bool
@ -40,7 +40,7 @@ class FileLibrary extends MyStorageLibrary
} }
} }
$saveFile = $fullPath . DIRECTORY_SEPARATOR . $this->getFileName(); $saveFile = $fullPath . DIRECTORY_SEPARATOR . $this->getFileName();
$this->_fileType = $this->getFileMimeType($saveFile); $this->_mimeType = mime_content_type($saveFile);
log_message("debug", "Storage Save-> " . $saveFile); log_message("debug", "Storage Save-> " . $saveFile);
return file_put_contents($saveFile, $content); return file_put_contents($saveFile, $content);
} }