diff --git a/app/Config/Routes.php b/app/Config/Routes.php index bb597a5..ff1f65b 100644 --- a/app/Config/Routes.php +++ b/app/Config/Routes.php @@ -30,14 +30,14 @@ $routes->group('mangboard', ['namespace' => 'App\Controllers\Mangboard'], functi $routes->cli('check_level', 'UserController::check_level'); $routes->cli('check_level/(:alpha)', 'UserController::check_level/$1'); }); - $routes->group('crawler', function ($routes) { - $routes->cli('yamap/(:any)', 'CrawlerController::yamap/$1'); - $routes->cli('yamap/(:any)/(:any)', 'CrawlerController::yamap/$1/$2'); - $routes->cli('yamoon/(:any)', 'CrawlerController::yamoon/$1'); - $routes->cli('yamoon/(:any)/(:any)', 'CrawlerController::yamoon/$1/$2'); - $routes->cli('sir/(:any)', 'CrawlerController::sir/$1'); - $routes->cli('sir/(:any)/(:any)', 'CrawlerController::sir/$1/$2'); - $routes->cli('inven/(:any)', 'CrawlerController::inven/$1'); - $routes->cli('inven/(:any)/(:any)', 'CrawlerController::inven/$1/$2'); + $routes->group('crawler', ['namespace' => 'App\Controllers\Mangboard\Crawler'], function ($routes) { + $routes->cli('yamap/(:any)', 'YamapCrawler::execute/$1'); + $routes->cli('yamap/(:any)/(:any)', 'YamapCrawler::execute/$1/$2'); + $routes->cli('yamoon/(:any)', 'YamoonCrawler::execute/$1'); + $routes->cli('yamoon/(:any)/(:any)', 'YamoonCrawler::execute/$1/$2'); + $routes->cli('sir/(:any)', 'SirCrawler::execute/$1'); + $routes->cli('sir/(:any)/(:any)', 'SirCrawler::execute/$1/$2'); + $routes->cli('inven/(:any)', 'InvenCrawler::execute/$1'); + $routes->cli('inven/(:any)/(:any)', 'InvenCrawler::execute/$1/$2'); }); }); diff --git a/app/Controllers/AuthController.php b/app/Controllers/AuthController.php new file mode 100644 index 0000000..17c9c23 --- /dev/null +++ b/app/Controllers/AuthController.php @@ -0,0 +1,58 @@ + LAYOUTS['empty'], + 'title' => '로그인', + 'forms' => [ + 'attributes' => ['action' => 'post', 'class' => 'row g-3'], + 'hiddens' => [RETURN_URL => session()->get(RETURN_URL)], + ] + ]; + return view('auth/login', $viewDatas); + } + + public function signin() + { + $id = $this->request->getVar('id'); + $passwd = $this->request->getVar('passwd'); + $model = new UserModel(); + $user = $model->asObject(UserEntity::class)->where('id', $id)->first(); + if (is_null($user) || !isset($user->passwd)) { + session()->setFlashdata('error', "사용자ID: {$id}가 존재하지 않습니다."); + return redirect()->back()->withInput(); + } + if (password_verify($passwd, $user->passwd)) { + //Session에 Login 정보전달 + $authData = [ + 'uid' => $user->uid, + 'name' => $user->name, + 'email' => $user->email, + 'role' => $user->role, + ISLOGIN => true + ]; + session()->set($authData); + return redirect()->to($this->request->getVar(RETURN_URL) ? $this->request->getVar(RETURN_URL) : "/"); + } else { + session()->setFlashdata('error', '암호가 맞지 않습니다.'); + return redirect()->back()->withInput(); + } + } + + public function logout() + { + //Session에 Login 정보 삭제 + session()->set([ISLOGIN => false]); + session_destroy(); + return redirect()->route('/'); + } +} diff --git a/app/Controllers/Cloudflare/AccountController.php b/app/Controllers/Cloudflare/AccountController.php new file mode 100644 index 0000000..597a394 --- /dev/null +++ b/app/Controllers/Cloudflare/AccountController.php @@ -0,0 +1,67 @@ +session = $this->loginCheck_AuthTrait(); + } + final public function getMySocket(string $email, $api_key): AccountSocket + { + if ($this->_mySocket === null) { + $this->_mySocket = new AccountSocket($email, $api_key); + } + return $this->_mySocket; + } + final public function getModel(): AccountModel + { + if ($this->_model === null) { + $this->_model = new AccountModel(); + } + return $this->_model; + } + public function create(string $email, $api_key, array $formDatas = []) + { + //전송 + $result = $this->getMySocket($email, $api_key)->create($email); + //답변형태 + // [ + // {"id":"078e88a7735965b661715af13031ecb0", + // "name":"Cloudwin002@idcjp.jp's Account", + // "type":"standard", + // "settings":{ + // "enforce_twofactor":false, + // "api_access_enabled":null, + // "access_approval_expiry":null, + // "use_account_custom_ns_by_default":false + // }, + // "legacy_flags":{"enterprise_zone_quota":{"maximum":0,"current":0,"available":0}}, + // "created_on":"2017-06-26T05:44:49.470184Z"} + // ] + $formDatas[$this->getModel()->getTitleField()] = $email; + $formDatas['key'] = $api_key; + $formDatas[$this->getModel()->PK()] = $result->id; + $formDatas[$this->getModel()->getTitleField()] = $result->name; + $formDatas['type'] = $result->type; + $formDatas['status'] = 'use'; + $formDatas['updated_at'] = $result->created_on; + $formDatas['created_at'] = $result->created_on; + $entity = $this->getModel()->create($formDatas); + log_message("notice", __FUNCTION__ . "=> {$entity->getTitle()} 생성을 완료하였습니다."); + } +} diff --git a/app/Controllers/CommonController.php b/app/Controllers/CommonController.php index 1ba4846..ff4bb11 100644 --- a/app/Controllers/CommonController.php +++ b/app/Controllers/CommonController.php @@ -7,7 +7,6 @@ use App\Controllers\BaseController; abstract class CommonController extends BaseController { private $_options = []; - final public function __get($name) { if (!array_key_exists($name, $this->_options)) { @@ -20,4 +19,65 @@ abstract class CommonController extends BaseController { $this->_options[$name] = $value; } + + //전송된 값 검증 및 임시저장 + final public function getFormFieldDatas(array $fields, array $fieldRules, array $formDatas = []): array + { + foreach ($fields as $field) { + $formDatas[$field] = rtrim($this->request->getVar($field)); + log_message("debug", "{$field} : {$formDatas[$field]}"); + } + //변경할 값 확인 + if (!$this->validate($fieldRules)) { + throw new \Exception("데이터 검증 오류발생\n" . implode("\n", $this->validator->getErrors())); + } + return $formDatas; + } + //Field별 Form Option용 + protected function getFormFieldOption(string $field, array $options): array + { + switch ($field) { + default: + $temps = lang($this->_className . '.' . strtoupper($field)); + if (!is_array($temps)) { + throw new \Exception(__FUNCTION__ . "에서 {$field}의 데이터가 array가 아닙니다.\n" . var_export($temps, true)); + } + $options[$field] = [ + ["" => lang($this->_className . '.label.' . $field) . ' 선택'], + lang($this->_className . '.' . strtoupper($field)) + ]; + break; + } + return $options; + } + //Field별 Form Option용 + final public function getFormFieldOptions(array $fields, array $options = []): array + { + foreach ($fields as $field) { + if (is_array($field)) { + throw new \Exception(__FUNCTION__ . "에서 field가 array 입니다.\n" . var_export($field, true)); + } + $options = $this->getFormFieldOption($field, $options); + } + return $options; + } + protected function getFormFieldRule(string $field, array $rules): array + { + if (is_array($field)) { + throw new \Exception(__FUNCTION__ . "=> field가 array 입니다.\n" . var_export($field, true)); + } + switch ($field) { + default: + $rules[$field] = $this->_model->getFieldRule($field, $rules);; + break; + } + return $rules; + } + final public function getFormFieldRules(array $fields, array $rules = []): array + { + foreach ($fields as $field) { + $rules = $this->getFormFieldRule($field, $rules); + } + return $rules; + } } diff --git a/app/Controllers/Mangboard/Crawler/InvenCrawler.php b/app/Controllers/Mangboard/Crawler/InvenCrawler.php new file mode 100644 index 0000000..8f99c19 --- /dev/null +++ b/app/Controllers/Mangboard/Crawler/InvenCrawler.php @@ -0,0 +1,175 @@ +extract(['data-src']); + if (count($attributes)) { + $url = $attributes[0]; + } + break; + case 'img': + default: + $url = parent::getUrlByMediaType($node, $media_type, $attr); + break; + } + return $url; + } + //작성내용 + //
+ //
+ // + //
+ //
+ // + //

+ //

+ //

+ //

+ //

+ //
+ //
^^
+ //
+ // + //
+ protected function detail_content_process(int $cnt, array $listInfo): array + { + $response = $this->getMySocket()->getContent($listInfo['detail_url']); + $selector = $this->getSelector($response, getenv("inven.view.content.tag")); + $formDatas = []; + $formDatas['image_path'] = ""; + $formDatas['content'] = $selector->html(); + //File DB 및 Board DB 등록작업등 + $this->getBoardModel()->createByCrawler( + $this->getBoardsEntity(), + $this->getUserEntity(), + $cnt, + $listInfo, + [], + $formDatas + ); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + return $listInfo; + } + protected function detail_download_process(int $cnt, array $listInfo): array + { + $response = $this->getMySocket()->getContent($listInfo['detail_url']); + $selector = $this->getSelector($response, getenv("inven.view.content.tag")); + + $media_urls = $this->getUrlsByMediaType($selector, "img", "src"); + $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); + if ($this->isDebug) { + throw new \Exception(sprintf( + "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", + __FUNCTION__, + var_export($listInfo, true), + var_export($media_urls, true) + )); + } else { + // Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리 + $storages = $this->media_process($media_urls); + if (!count($storages)) { + throw new \Exception("등록할 자료가 없습니다."); + } + $this->backend_process($cnt, $listInfo, $storages); + } + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + return $listInfo; + } + //리스트내용 + //
+ // + // + // + // + // + // + // + // + // + //
1589 + //
+ // + // [1] + // 사진 + //
+ // 		+ // 레벨 아이콘 + // 배수민 + // 09-151,5021
+ //
+ public function execute(string $board_name, string $user_id = null, ...$params): void + { + try { + //추가옵션 + $this->isDebug = in_array('debug', $params); + $this->isCopy = in_array('copy', $params); + $this->setBoardName($board_name); + $this->login_process($user_id); + //실행 + $listInfos = []; + if ($this->isDebug) { + $listInfo = []; + $listInfo['title'] = 'test_title'; + $listInfo['nickname'] = 'test_name'; + $listInfo['hit'] = 1; + $listInfo['date'] = date("Y-m-d H:i:s"); + $listInfo['detail_url'] = getenv("inven.view.test.url.{$this->getBoardName()}"); + $listInfos[] = $listInfo; + } else { + $response = $this->getMySocket()->getContent(getenv("inven.list.url.{$this->getBoardName()}")); + $this->getSelector($response, getenv("inven.list.tag.{$this->getBoardName()}"))->each( + function (Crawler $node) use (&$listInfos): void { + $hit = $node->filter(getenv("inven.list.item.hit.tag"))->text(); + $date = date("Y") . "-" . $node->filter(getenv("inven.list.item.date.tag"))->text(); + $nickname = $node->filter(getenv("inven.list.item.nickname.tag"))->text(); + //작성자가 "관리자"가 아닌 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 + $link_node = $node->filter(getenv("inven.list.item.link.tag")); + $detail_url = $link_node->attr("href"); + $title = $link_node->text(); + $listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit]; + } + ); + } + if (!count($listInfos)) { + throw new \Exception("Target URL이 없습니다."); + } + $this->list_process(intval(getenv("inven.list.max_limit.{$this->getBoardName()}")), $listInfos); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + } catch (\Exception $e) { + log_message("warning", sprintf( + "\n---%s 오류---\n%s\n-----------------------------------------\n", + __FUNCTION__, + $e->getMessage() + )); + } + } +} diff --git a/app/Controllers/Mangboard/Crawler/YamapCrawler.php b/app/Controllers/Mangboard/Crawler/YamapCrawler.php new file mode 100644 index 0000000..d5272e7 --- /dev/null +++ b/app/Controllers/Mangboard/Crawler/YamapCrawler.php @@ -0,0 +1,141 @@ +getMySocket()->getContent($listInfo['detail_url']); + $selector = $this->getSelector($response, getenv("yamap.view.content.tag")); + $formDatas = []; + $formDatas['image_path'] = ""; + $formDatas['content'] = $selector->html(); + //File DB 및 Board DB 등록작업등 + $this->getBoardModel()->createByCrawler( + $this->getBoardsEntity(), + $this->getUserEntity(), + $cnt, + $listInfo, + [], + $formDatas + ); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + return $listInfo; + } + protected function detail_download_process(int $cnt, array $listInfo): array + { + $response = $this->getMySocket()->getContent($listInfo['detail_url']); + $selector = $this->getSelector($response, getenv("yamap.view.content.tag")); + $media_urls = $this->getUrlsByMediaType($selector, "img", "src"); + $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); + if ($this->isDebug) { + throw new \Exception(sprintf( + "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", + __FUNCTION__, + var_export($listInfo, true), + var_export($media_urls, true) + )); + } else { + // Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리 + $storages = $this->media_process($media_urls); + if (!count($storages)) { + throw new \Exception("등록할 자료가 없습니다."); + } + $this->backend_process($cnt, $listInfo, $storages); + } + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + return $listInfo; + } + //리스트내용 + //
+ //
요즘 패션
+ //
+ // + // 괴강고귀 + // + // + // | 추천 (14) | 조회 (432) + // + //
+ //
+ // + // + // + // 2024-09-14 01:53:45 + // + //
+ //
+ //
+ //

+ //

 

+ //
+ //
+ //
+ // + // + //
+ //
+ //
+ public function execute(string $board_name, string $user_id = null, ...$params): void + { + try { + //추가옵션 + $this->isDebug = in_array('debug', $params); + $this->isCopy = in_array('copy', $params); + $this->setBoardName($board_name); + $this->login_process($user_id); + //실행 + $listInfos = []; + if ($this->isDebug) { + $listInfo = []; + $listInfo['title'] = 'test_title'; + $listInfo['nickname'] = 'test_name'; + $listInfo['hit'] = 1; + $listInfo['date'] = date("Y-m-d H:i:s"); + $listInfo['detail_url'] = getenv("yamap.view.test.url.{$this->getBoardName()}"); + $listInfos[] = $listInfo; + } else { + $response = $this->getMySocket()->getContent(getenv("yamap.list.url.{$this->getBoardName()}")); + $selector = $this->getSelector($response, getenv("inven.list.tag.{$this->getBoardName()}")); + $selector->filter(getenv("yamap.list.item.tag"))->each( + function (Crawler $node) use (&$listInfos): void { + $hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text(); + $date = date("Y") . "-" . $node->filter(getenv("yamap.list.item.date.tag"))->text(); + $nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text(); + //작성자가 "관리자"가 아닌 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 + $link_node = $node->filter(getenv("yamap.list.item.link.tag")); + $detail_url = $link_node->attr("href"); + $title = $link_node->text(); + $listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit]; + } + ); + } + if (!count($listInfos)) { + throw new \Exception("Target URL이 없습니다."); + } + $this->list_process(intval(getenv("yamap.list.max_limit.{$this->getBoardName()}")), $listInfos); + log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); + } catch (\Exception $e) { + log_message("warning", sprintf( + "\n---%s 오류---\n%s\n-----------------------------------------\n", + __FUNCTION__, + $e->getMessage() + )); + } + } +} diff --git a/app/Controllers/Mangboard/CrawlerController.php b/app/Controllers/Mangboard/CrawlerController.php index 6018cd4..ffd7b13 100644 --- a/app/Controllers/Mangboard/CrawlerController.php +++ b/app/Controllers/Mangboard/CrawlerController.php @@ -3,16 +3,81 @@ namespace App\Controllers\Mangboard; use App\Controllers\CommonController; +use App\Entities\Mangboard\BoardsEntity; use App\Entities\Mangboard\UserEntity; -use App\Libraries\MyCrawler\Mangboard\InvenCrawler; -use App\Libraries\MyCrawler\Mangboard\SirCrawler; -use App\Libraries\MyCrawler\Mangboard\YamapCrawler; -use App\Libraries\MyCrawler\Mangboard\YamoonCrawler; +use App\Libraries\MySocket\WebSocket; +use App\Libraries\MyStorage\MangboardStorage; +use App\Models\Mangboard\BoardModel; +use App\Models\Mangboard\BoardsModel; use App\Models\Mangboard\UserModel; +use App\Traits\FileTrait; +use CodeIgniter\HTTP\RequestInterface; +use CodeIgniter\HTTP\ResponseInterface; +use Psr\Log\LoggerInterface; +use Symfony\Component\DomCrawler\Crawler; -class CrawlerController extends CommonController +abstract class CrawlerController extends CommonController { - private $_userModel = null; + use FileTrait; + private $_mySocket = null; + private $_board_model = null; + private $_user_model = null; + private $_user_entity = null; + private $_boards_entity = null; + private $_board_name = ""; + public function initController(RequestInterface $request, ResponseInterface $response, LoggerInterface $logger) + { + parent::initController($request, $response, $logger); + } + abstract protected function detail_content_process(int $cnt, array $listInfo): array; + abstract protected function detail_download_process(int $cnt, array $listInfo): array; + abstract protected function getHost(): string; + final protected function getBoardName(): string + { + return $this->_board_name; + } + final protected function setBoardName(string $board_name): void + { + $this->_board_name = $board_name; + } + final protected function getUserEntity(): UserEntity + { + return $this->_user_entity; + } + final protected function setUserEntity(UserEntity $user_entity): void + { + $this->_user_entity = $user_entity; + } + //-----------------------필수항목-------------------// + final protected function getMySocket() + { + if ($this->_mySocket === null) { + $this->_mySocket = new WebSocket($this->getHost()); + } + return $this->_mySocket; + } + final protected function createMyStorage(): MangboardStorage + { + return new MangboardStorage($this->getBoardName(), $this->getUserEntity()); + } + final protected function getBoardsEntity(): BoardsEntity + { + if ($this->_boards_entity === null) { + $boardsModel = new BoardsModel(); + $this->_boards_entity = $boardsModel->getEntityByID($this->getBoardName()); + if ($this->_boards_entity === null) { + throw new \Exception(__FUNCTION__ . "=> {$this->getBoardName()}에 해당 Board 정보가 존재하지 않습니다."); + } + } + return $this->_boards_entity; + } + final protected function getBoardModel(): BoardModel + { + if ($this->_board_model === null) { + $this->_board_model = new BoardModel("mb_" . $this->getBoardName()); + } + return $this->_board_model; + } public function getUserModel(): UserModel { if ($this->_user_model === null) { @@ -20,14 +85,13 @@ class CrawlerController extends CommonController } return $this->_user_model; } - public function login(string $id = ""): bool|UserEntity + protected function login_process(string $user_id = null): void { - $host = getenv("mangboard.host.url"); - $id = $id == "" ? getenv("mangboard.login.default.id") : $id; - $password = getenv("mangboard.login.default.password"); - $user_entity = $this->getUserModel()->getEntityByID($id); + $user_id = $user_id ?? getenv("mangboard.login.default.id"); + $password = getenv("mangboard.login.default.password"); + $this->setUserEntity($this->getUserModel()->getEntityByID($user_id)); // $response = $this->getWebLibrary($host)->getResponse( - // $host . getenv("mangboard.login.url"), + // getenv("mangboard.host.url") . getenv("mangboard.login.url"), // "post", // [ // 'form_params' => [ @@ -44,71 +108,177 @@ class CrawlerController extends CommonController // } else { // throw new \Exception("연결실패:" . $response->getStatusCode()); // } - log_message("notice", "{$id}로 로그인 성공"); - return $user_entity; + log_message("notice", "{$user_id}로 로그인 성공"); } - public function yamap(string $board_name, ...$params): string + + final protected function getSelector(string $content, string $tag): Crawler { - try { - //1. 사이트 로그인 처리 - $user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); - //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. - $crawler = new YamapCrawler(getenv('yamap.host.url'), $board_name, $user_entity); - $crawler->isDebug = in_array('debug', $params); - $crawler->isCopy = in_array('copy', $params); - $crawler->execute(); - return "완료되었습니다."; - } catch (\Exception $e) { - log_message("error", $e->getMessage()); - return $e->getMessage(); + $crawler = new Crawler($content); + if ($this->isDebug) { + log_message("debug", __FUNCTION__ . "=> " . $tag); } + $crawler->filter($tag); + if ($this->isDebug) { + log_message("debug", sprintf( + "\n------------%s HTML-------------\n%s\n-----------------------------------------------------\n", + __FUNCTION__, + $crawler->filter($tag)->html() + )); + } + return $crawler->filter($tag); } - public function yamoon(string $board_name, ...$params): string + protected function changeURLByCrawler(string $url): string { - try { - //1. 사이트 로그인 처리 - $user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); - //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. - $crawler = new YamoonCrawler(getenv("yamoon.host.url"), $board_name, $user_entity); - $crawler->isDebug = in_array('debug', $params); - $crawler->isCopy = in_array('copy', $params); - $crawler->execute(); - return "완료되었습니다."; - } catch (\Exception $e) { - log_message("error", $e->getMessage()); - return $e->getMessage(); - } + return preg_match('/^[^?]+/', $url, $matches) ? $matches[0] : null; } - public function sir(string $board_name, ...$params): string + protected function getUrlByMediaType(Crawler $node, string $media_tag, string $attr): null|string { - try { - //1. 사이트 로그인 처리 - $user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); - //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. - $crawler = new SirCrawler(getenv("sir.host.url"), $board_name, $user_entity); - $crawler->isDebug = in_array('debug', $params); - $crawler->isCopy = in_array('copy', $params); - $crawler->execute(); - return "완료되었습니다."; - } catch (\Exception $e) { - log_message("error", $e->getMessage()); - return $e->getMessage(); + switch ($media_tag) { + case 'video': + try { + $url = $node->attr($attr); // 또는 + } catch (\Exception) { + $url = $node->children()->attr("src"); //