idcjp/Automation
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automation init...1

Browse Source
This commit is contained in:
최준흠 2024-09-08 03:51:23 +09:00
parent 84245e4c0b
commit 40e47b2815
5 changed files with 118 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
app/Controllers/CLI/Crawler.php
View File

@ -4,10 +4,6 @@ namespace App\Controllers\CLI;
use App\Controllers\BaseController;
use App\Libraries\MyCrawler\YamapLibrary;
use App\Libraries\Mangboard\BoardLibrary;
use App\Models\Mangboard\UserModel;
use App\Models\Mangboard\BoardModel;
use App\Entities\Mangboard\BoardEntity;
class Crawler extends BaseController
{
@ -15,44 +11,9 @@ class Crawler extends BaseController
{
try {
$isDebug = in_array("debug", $params);
//1. 사이트 로그인 처리
$userModel = new UserModel();
$user = $userModel->getEntityByID("idcjp");
// if (!in_array("skip_login", $params)) {
// $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url'));
// $daemonidc->setDebug($isDebug);
// $daemonidc->login(
// getenv('daemonidc.login.url'),
// getenv('daemonidc.login.user_id'),
// getenv('daemonidc.login.user_password')
// );
// }
//2.Yamap사이트에서 자유게시판에서 최근 게시물 데이터 가져오기
$yamap = new YamapLibrary();
$yamap->setDebug($isDebug);
list($item, $fileInfos) = $yamap->execute();
//3.망보드 일반게시판에 게시물 등록 처리
if (!in_array("skip_create", $params)) {
$board = new BoardLibrary(new BoardModel(getenv("crawler.yamap.registration.table")));
$board->setDebug($isDebug);
//미디어관련정보 entity에 넣기
$entity = new BoardEntity();
$entity->title = $item["title"];
$entity->user_name = $item["nickname"];
$entity->reg_date = $item['time'];
$entity->hit = $item['hit'];
$entity->data_type = "html";
$entity->editor_type = "S";
foreach ($fileInfos as $fileInfo) {
$entity->content .= $fileInfo["mediatag"];
}
//망보드에 넣기
$board->create($entity);
}
$yamap->execute();
log_message("notice", "Crawler->" . __FUNCTION__ . " 작업이 완료되었습니다.");
return true;
} catch (\Exception $e) {

31
app/Libraries/Mangboard/FileLibrary.php
View File

@ -11,8 +11,10 @@ use App\Entities\Mangboard\FileEntity;
class FileLibrary extends MyStorageLibrary
{
private $_user = null;
private $_boardName = null;
private $_boardName = "";
private $_boardTable = "";
private $_model = null;
private $_fileEntity = null;
public function __construct(string $path)
{
parent::__construct($path);
@ -39,6 +41,17 @@ class FileLibrary extends MyStorageLibrary
{
$this->_boardName = $boardName;
}
public function getBoardTable(): string
{
if ($this->_boardTable === null) {
throw new \Exception("BoardModel이 지정되지 않았습니다.");
}
return $this->_boardTable;
}
public function setBoardTable(string $boardTable): void
{
$this->_boardName = $boardTable;
}
public function getModel(): FileModel
{
@ -48,22 +61,32 @@ class FileLibrary extends MyStorageLibrary
return $this->_model;
}
//망보드 파일관리 table에 등록
public function createFile($content): bool
public function getFileEntity(): null|FileEntity
{
return $this->_fileEntity;
}
public function save($content): bool
{
if (!parent::save($content)) {
return false;
}
//망보드 파일관리 table에 등록
try {
//mb_files 모델작업
$entity = new FileEntity();
$entity->setTitle($this->getFileName());
$entity->user_pid = $this->getUser()->getPK();
$entity->user_name = $this->getUser()->getTitle();
$entity->board_name = $this->getBoardName();
$entity->table_name = "mb_" . $this->getBoardName();
$entity->table_name = $this->getBoardTable();
$entity->file_path = $this->getPath();
$entity->file_type = $this->getMimeType();
$entity->reg_date = date("Y-m-d H:i:s");
$this->_fileEntity = $this->getModel()->create($entity);
return true;
} catch (\Exception $e) {
return false;
}
}
}

4
app/Libraries/MyCrawler/MyCrawlerLibrary.php
View File

@ -13,7 +13,7 @@ abstract class MyCrawlerLibrary extends CommonLibrary
}
abstract protected function getMySocket();
abstract protected function getMyStorage();
abstract public function execute(): array;
abstract public function execute(): void;
final protected function getContent(string $url, string $tag): Crawler
{
@ -75,7 +75,7 @@ abstract class MyCrawlerLibrary extends CommonLibrary
}
return $downloadInfos;
}
final protected function save(string $mediaType, array $downloadInfos, $fileInfos = []): array
protected function save(string $mediaType, array $downloadInfos, $fileInfos = []): array
{
foreach ($downloadInfos as $downloadInfo) {
$this->getMyStorage()->setFileName($downloadInfo['fileName']);

83
app/Libraries/MyCrawler/YamapLibrary.php
View File

@ -2,34 +2,58 @@
namespace App\Libraries\MyCrawler;
use App\Libraries\Mangboard\FileLibrary as MyStorage;
use App\Libraries\MySocket\WebLibrary as MySocket;
use Symfony\Component\DomCrawler\Crawler;
use App\Libraries\MySocket\WebLibrary as MySocket;
use App\Libraries\Mangboard\FileLibrary as MyStorage;
use App\Libraries\Mangboard\BoardLibrary;
use App\Models\Mangboard\BoardModel;
use App\Entities\Mangboard\BoardEntity;
use App\Models\Mangboard\UserModel;
class YamapLibrary extends MyCrawlerLibrary
{
private $_mySocket = null;
private $_myStorage = null;
private $_boardModel = null;
public function __construct()
{
parent::__construct();
}
protected function getMySocket()
final public function getMySocket(): mixed
{
if ($this->_mySocket === null) {
$this->_mySocket = new MySocket(getenv('yamap.host.url'));
}
return $this->_mySocket;
}
final public function setMySocket($mySocket): void
{
$this->_mySocket = $mySocket;
}
protected function getMyStorage()
final public function getMyStorage(): mixed
{
if ($this->_myStorage === null) {
$this->_myStorage = new MyStorage(getenv('yamap.storage.upload.path'));
$this->_myStorage->setBoardName(getenv('crawler.yamap.registration.board'));
$this->_myStorage->setBoardTable($this->getBoardModel()->getTable());
}
return $this->_myStorage;
}
final public function setMyStorage($myStorage): void
{
$this->_myStorage = $myStorage;
}
private function getBoardModel(): BoardModel
{
if ($this->_boardModel === null) {
$this->_boardModel = new BoardModel(getenv("crawler.yamap.registration.table"));
}
return $this->_boardModel;
}
private function mainPage(string $url): array
{
@ -57,7 +81,7 @@ class YamapLibrary extends MyCrawlerLibrary
return $items;
}
private function detailPage(string $url): array
private function detailPage(string $url, array $fileInfos = []): array
{
$crawler = $this->getContent($url, getenv("yamap.view.content.tag"));
//3. Image 처리
@ -65,13 +89,44 @@ class YamapLibrary extends MyCrawlerLibrary
$fileInfos = $this->save("image", $downloadInfos);
//4. Video(mp4) 처리
$downloadInfos = $this->download($crawler, ["tag" => "video", "attr" => "src"]);
return $this->save("video", $downloadInfos);
return $this->save("video", $downloadInfos, $fileInfos);
}
public function execute(): array
private function createBoard(array $item, array $fileInfos): BoardEntity
{
$board = new BoardLibrary($this->getBoardModel());
$board->setDebug($this->getDebug());
//미디어관련정보 entity에 넣기
$entity = new BoardEntity();
$entity->title = $item["title"];
$entity->user_name = $item["nickname"];
$entity->reg_date = $item['time'];
$entity->hit = $item['hit'];
$entity->data_type = "html";
$entity->editor_type = "S";
foreach ($fileInfos as $fileInfo) {
$entity->content .= $fileInfo["mediatag"];
}
//망보드에 넣기
return $board->create($entity);
}
public function execute(): void
{
//1. 사이트 로그인 처리
$userModel = new UserModel();
$user = $userModel->getEntityByID("idcjp");
// $daemonidc = new MyWebLibrary(getenv('daemonidc.host.url'));
// $daemonidc->setDebug($isDebug);
// $daemonidc->login(
// getenv('daemonidc.login.url'),
// getenv('daemonidc.login.user_id'),
// getenv('daemonidc.login.user_password')
// );
$this->getMyStorage()->setUser($user);
$items = [];
//1. 해당사이트 MainPage 처리
//2. 해당사이트 MainPage 처리
if ($this->getDebug()) {
$items[] = [
'title' => getenv("yamap.view.test.title"),
@ -83,8 +138,14 @@ class YamapLibrary extends MyCrawlerLibrary
} else {
$items = $this->mainPage(getenv("yamap.list.url"));
}
//2. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리
$fileInfos = $this->detailPage($items[0]["detail_url"]);
return array($items[0], $fileInfos);
if (!count($items)) {
throw new \Exception("Yamap 사이트에서 게시물이 존재하지 않습니다.");
}
//최초 게시물만 등록하기 위함
$item = $items[0];
//3. DetailPage 처리 : bbs_view > div.contents 가진 객체를 찾아서 처리
$fileInfos = $this->detailPage($item["detail_url"]);
//4.망보드 일반게시판에 게시물 등록 처리
$this->createBoard($item, $fileInfos);
}
}

8
app/Libraries/MyStorage/FileLibrary.php
View File

@ -40,8 +40,12 @@ class FileLibrary extends MyStorageLibrary
}
}
$saveFile = $fullPath . DIRECTORY_SEPARATOR . $this->getFileName();
$this->_mimeType = mime_content_type($saveFile);
log_message("debug", "Storage Save-> " . $saveFile);
return file_put_contents($saveFile, $content);
if (!file_put_contents($saveFile, $content)) {
return false;
}
//File형식에 따른 MimeType 지정
$this->_mimeType = mime_content_type($saveFile);
return true;
}
}