#include "Base/Database.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include bool isUrlValid(boost::asio::io_context &ioContext, const std::string &host, const std::string &port, const std::string &target); int main(int argc, char const *argv[]) { boost::program_options::options_description description("Allowed options"); // clang-format off description.add_options() ("help,h", "produce help message.") ("database,d", boost::program_options::value(),"set database path") ("delete-invalid", boost::program_options::value()->default_value(false),"delete invalid url"); // clang-format on boost::program_options::variables_map values; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, description), values); boost::program_options::notify(values); std::string path; if (values.count("help")) { std::cout << description << std::endl; std::exit(0); } else if (values.count("database")) { path = values.at("database").as(); } if (path.empty()) { std::cerr << "please specify the database path." << std::endl; std::cout << description << std::endl; return 1; } else if (!std::filesystem::exists(path)) { std::cerr << "database file " << path << " not existed." << std::endl; return 2; } Older::Database database; if (!database.open(path)) { return 3; } boost::asio::io_context ioContext; auto items = database.visitRecords(); for (auto &item : items) { bool valid = isUrlValid(ioContext, "amass.fun", "443", item.url); std::cout << item.url << std::endl; std::cout << "valid: " << valid << std::endl; if (!valid && values.at("delete-invalid").as()) { std::cout << "delete: " << database.removeVisitRecord(item.id) << std::endl; } std::cout << "----------" << std::endl; } return 0; } bool isUrlValid(boost::asio::io_context &ioContext, const std::string &host, const std::string &port, const std::string &target) { using namespace boost; using namespace boost::asio; using namespace boost::asio::ip; using namespace boost::beast; try { // 1. 创建SSL上下文 ssl::context ssl_ctx(ssl::context::tlsv12_client); ssl_ctx.set_default_verify_paths(); ssl_ctx.set_verify_mode(ssl::verify_peer); // 2. 创建TCP解析器和SSL流 tcp::resolver resolver(ioContext); beast::ssl_stream stream(ioContext, ssl_ctx); // 3. 设置SNI主机名(重要!) if (!SSL_set_tlsext_host_name(stream.native_handle(), host.c_str())) { throw boost::system::system_error(::ERR_get_error(), boost::asio ::error::get_ssl_category()); } // 4. 解析主机名并建立连接 auto const results = resolver.resolve(host, port); beast::get_lowest_layer(stream).connect(results); stream.handshake(ssl::stream_base::client); // 5. 构造并发送HEAD请求 http::request req{http::verb::get, target, 11}; req.set(http::field::host, host); req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); http::write(stream, req); // 6. 读取响应 beast::flat_buffer buffer; http::response res; http::read(stream, buffer, res); // 7. 检查HTTP状态码(2xx/3xx视为可访问) const unsigned status = res.result_int(); const bool accessible = (status >= 200 && status < 400); // 8. 优雅关闭连接 beast::error_code ec; stream.shutdown(ec); if (ec == net::error::eof || ec == boost::asio::ssl::error::stream_truncated) { ec = {}; } return accessible; } catch (const std::exception &e) { std::cerr << "Error: " << e.what() << std::endl; return false; } }