利用stream实现一个简单的http下载器"}（利用stream实现一个简单的http下载器"}）

其实这个http下载器的功能已经相当完善了，支持：限速、post投递和上传、自定义http header、设置user agent、设置range和超时

而且它还不单纯只能下载http，由于使用了stream，所以也支持其他协议，你也可以用它来进行文件之间的copy、纯tcp下载等等。。

完整demo请参考：https://github.com/waruqi/tbox/wiki

stream.c

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 /* ////////////////////////////////////////////////////////////////////////////////////// * includes */ #include "../demo.h" /* ////////////////////////////////////////////////////////////////////////////////////// * types */ typedef struct __tb_demo_context_t { // verbose tb_bool_t verbose; }tb_demo_context_t; /* ////////////////////////////////////////////////////////////////////////////////////// * func */ #ifdef TB_CONFIG_MODULE_HAVE_OBJECT static tb_bool_t tb_demo_http_post_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv) { // percent tb_size_t percent = 0; if (size > 0) percent = (tb_size_t)((offset * 100) / size); else if (state == TB_STATE_CLOSED) percent = 100; // trace tb_trace_i("post: %llu, rate: %lu bytes/s, percent: %lu%%, state: %s", save, rate, percent, tb_state_cstr(state)); // ok return tb_true; } static tb_bool_t tb_demo_stream_head_func(tb_char_t const* line, tb_cpointer_t priv) { tb_printf("response: %s\n", line); return tb_true; } static tb_bool_t tb_demo_stream_save_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv) { // check tb_demo_context_t* context = (tb_demo_context_t*)priv; tb_assert_and_check_return_val(context, tb_false); // print verbose info if (context->verbose) { // percent tb_size_t percent = 0; if (size > 0) percent = (tb_size_t)((offset * 100) / size); else if (state == TB_STATE_CLOSED) percent = 100; // trace tb_printf("save: %llu bytes, rate: %lu bytes/s, percent: %lu%%, state: %s\n", save, rate, percent, tb_state_cstr(state)); } // ok return tb_true; } /* ////////////////////////////////////////////////////////////////////////////////////// * globals */ static tb_option_item_t g_options[] = { {'-', "gzip", TB_OPTION_MODE_KEY, TB_OPTION_TYPE_BOOL, "enable gzip" } , {'-', "no-verbose", TB_OPTION_MODE_KEY, TB_OPTION_TYPE_BOOL, "disable verbose info" } , {'d', "debug", TB_OPTION_MODE_KEY, TB_OPTION_TYPE_BOOL, "enable debug info" } , {'k', "keep-alive", TB_OPTION_MODE_KEY, TB_OPTION_TYPE_BOOL, "keep alive" } , {'h', "header", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_CSTR, "the custem http header" } , {'-', "post-data", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_CSTR, "set the post data" } , {'-', "post-file", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_CSTR, "set the post file" } , {'-', "range", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_CSTR, "set the range" } , {'-', "timeout", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_INTEGER, "set the timeout" } , {'-', "limitrate", TB_OPTION_MODE_KEY_VAL, TB_OPTION_TYPE_INTEGER, "set the limitrate" } , {'h', "help", TB_OPTION_MODE_KEY, TB_OPTION_TYPE_BOOL, "display this help and exit"} , {'-', "url", TB_OPTION_MODE_VAL, TB_OPTION_TYPE_CSTR, "the url" } , {'-', tb_null, TB_OPTION_MODE_MORE, TB_OPTION_TYPE_NONE, tb_null } }; /* ////////////////////////////////////////////////////////////////////////////////////// * main */ tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv) { // done tb_option_ref_t option = tb_null; tb_stream_ref_t istream = tb_null; tb_stream_ref_t ostream = tb_null; tb_stream_ref_t pstream = tb_null; do { // init option option = tb_option_init("stream", "the stream demo", g_options); tb_assert_and_check_break(option); // done option if (tb_option_done(option, argc - 1, &argv[1])) { // debug & verbose tb_bool_t debug = tb_option_find(option, "debug"); tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true; // done url if (tb_option_find(option, "url")) { // init istream istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url")); tb_assert_and_check_break(istream); // ctrl http if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP) { // enable gzip? if (tb_option_find(option, "gzip")) { // auto unzip if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break; // need gzip if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break; } // enable debug? if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break; // custem header? if (tb_option_find(option, "header")) { // init tb_string_t key; tb_string_t val; tb_string_init(&key); tb_string_init(&val); // done tb_bool_t k = tb_true; tb_char_t const* p = tb_option_item_cstr(option, "header"); while (*p) { // is key? if (k) { if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++); else if (*p == ':') { // skip ':' p++; // skip space while (*p && tb_isspace(*p)) p++; // is val now k = tb_false; } else p++; } // is val? else { if (*p != ';') tb_string_chrcat(&val, *p++); else { // skip ';' p++; // skip space while (*p && tb_isspace(*p)) p++; // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // is key now k = tb_true; // clear key & val tb_string_clear(&key); tb_string_clear(&val); } } } // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // exit tb_string_exit(&key); tb_string_exit(&val); } // keep alive? if (tb_option_find(option, "keep-alive")) { if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break; } // post-data? if (tb_option_find(option, "post-data")) { tb_char_t const* post_data = tb_option_item_cstr(option, "post-data"); tb_hize_t post_size = tb_strlen(post_data); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (debug) tb_printf("post: %llu\n", post_size); } // post-file? else if (tb_option_find(option, "post-file")) { tb_char_t const* url = tb_option_item_cstr(option, "post-file"); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (debug) tb_printf("post: %s\n", url); } } // set range if (tb_option_find(option, "range")) { tb_char_t const* p = tb_option_item_cstr(option, "range"); if (p) { // the bof tb_hize_t eof = 0; tb_hize_t bof = tb_atoll(p); while (*p && tb_isdigit(*p)) p++; if (*p == '-') { p++; eof = tb_atoll(p); } if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break; } } // set timeout if (tb_option_find(option, "timeout")) { tb_size_t timeout = tb_option_item_uint32(option, "timeout"); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break; } // print verbose info if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url")); // open istream if (!tb_stream_open(istream)) { // print verbose info if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream))); break; } // print verbose info if (verbose) tb_printf("open: ok\n"); // init ostream if (tb_option_find(option, "more0")) { // the path tb_char_t const* path = tb_option_item_cstr(option, "more0"); // init ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (verbose) tb_printf("save: %s\n", path); } else { // the name tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/'); if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\'); if (!name) name = "/stream.file"; // the path tb_char_t path[TB_PATH_MAXN] = {0}; if (tb_directory_curt(path, TB_PATH_MAXN)) tb_strcat(path, name); else break; // init file ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (verbose) tb_printf("save: %s\n", path); } tb_assert_and_check_break(ostream); // the limit rate tb_size_t limitrate = 0; if (tb_option_find(option, "limitrate")) limitrate = tb_option_item_uint32(option, "limitrate"); // save it tb_hong_t save = 0; tb_demo_context_t context = {0}; context.verbose = verbose; if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break; } else tb_option_help(option); } else tb_option_help(option); } while (0); // exit pstream if (pstream) tb_stream_exit(pstream); pstream = tb_null; // exit istream if (istream) tb_stream_exit(istream); istream = tb_null; // exit ostream if (ostream) tb_stream_exit(ostream); ostream = tb_null; // exit option if (option) tb_option_exit(option); option = tb_null; return 0; } #else tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv) { return 0; } #endif