python Archives - 俺的BLOG

python中的变量&浅拷贝&深拷贝

Python中，万物皆对象！

即使是int, string, list这些在其他语言里的值类型，在python中也是对象。所以，变量简单的赋值，只是复制了引用地址，一个变量的改变会影响另一个变量。

int_a = [123]
int_b = int_a
print(int_a, int_b)
print(id(int_a), id(int_b))

int_b[0] = 456 # 修改int_b时，int_a也被修改了
print(int_a, int_b)
print(id(int_a), id(int_b))

# 输出内容：
[123] [123]
1996012610120 1996012610120
[456] [456]
1996012610120 1996012610120

int_a = [123]

int_b = int_a

print(int_a, int_b)

print(id(int_a), id(int_b))

int_b[0] = 456 # 修改int_b时，int_a也被修改了

print(int_a, int_b)

print(id(int_a), id(int_b))

# 输出内容：

[123] [123]

1996012610120 1996012610120

[456] [456]

1996012610120 1996012610120

浅拷贝

Python中的浅拷贝，只会拷贝最顶层变量的基本内容，若顶层变量中存在其他子对象引用，则子对象是以引用方式传递给新变量，两个变量是浅拷贝赋值关系时，修改子对象时，两个变量内容都会改变。

list_b = copy.copy(list_a)
print(list_a, list_b)
print(id(list_a), id(list_b))  # 两变量在内存中的地址不同

list_a[0] = 5  # 顶层变量各自独立，修改时互不影响
print(list_a, list_b)
print(id(list_a), id(list_b))

list_a[2].append(6)  # 子对象依然是引用传递，修改时有影响
print(list_a, list_b)
print(id(list_a), id(list_b))

# 输出内容：
[1, 2, [3, 4]] [1, 2, [3, 4]]
2756700693128 2756746622472
[5, 2, [3, 4]] [1, 2, [3, 4]]
2756700693128 2756746622472
[5, 2, [3, 4, 6]] [1, 2, [3, 4, 6]]
2756700693128 2756746622472

list_b = copy.copy(list_a)

print(list_a, list_b)

print(id(list_a), id(list_b)) # 两变量在内存中的地址不同

list_a[0] = 5 # 顶层变量各自独立，修改时互不影响

print(list_a, list_b)

print(id(list_a), id(list_b))

list_a[2].append(6) # 子对象依然是引用传递，修改时有影响

print(list_a, list_b)

print(id(list_a), id(list_b))

# 输出内容：

[1, 2, [3, 4]] [1, 2, [3, 4]]

2756700693128 2756746622472

[5, 2, [3, 4]] [1, 2, [3, 4]]

2756700693128 2756746622472

[5, 2, [3, 4, 6]] [1, 2, [3, 4, 6]]

2756700693128 2756746622472

深度拷贝

Python中的深度拷贝，会迭代拷贝所有子对象，两个变量是深度拷贝赋值关系时，修改任意一个变量互不影响。

import copy

list_a = [1, 2, [3, 4]]
list_b = copy.deepcopy(list_a)
print(list_a, list_b)
print(id(list_a), id(list_b))  # 两变量在内存中的地址不同

list_a[0] = 5  # 所有变量各自独立，修改时互不影响
print(list_a, list_b)
print(id(list_a), id(list_b))

list_a[2].append(6)  # 子对象也各自独立，修改时互不影响
print(list_a, list_b)
print(id(list_a), id(list_b))

# 输出结果：
[1, 2, [3, 4]] [1, 2, [3, 4]]
2447876842120 2447926113736
[5, 2, [3, 4]] [1, 2, [3, 4]]
2447876842120 2447926113736
[5, 2, [3, 4, 6]] [1, 2, [3, 4]]
2447876842120 2447926113736

import copy

list_a = [1, 2, [3, 4]]

list_b = copy.deepcopy(list_a)

print(list_a, list_b)

print(id(list_a), id(list_b)) # 两变量在内存中的地址不同

list_a[0] = 5 # 所有变量各自独立，修改时互不影响

print(list_a, list_b)

print(id(list_a), id(list_b))

list_a[2].append(6) # 子对象也各自独立，修改时互不影响

print(list_a, list_b)

print(id(list_a), id(list_b))

# 输出结果：

[1, 2, [3, 4]] [1, 2, [3, 4]]

2447876842120 2447926113736

[5, 2, [3, 4]] [1, 2, [3, 4]]

2447876842120 2447926113736

[5, 2, [3, 4, 6]] [1, 2, [3, 4]]

2447876842120 2447926113736

mqtt

mqtt实现多客户端实时通信，如平台推送消息至移动应用终端。

MQTT 它是 IBM 公司开发的一个即时通讯协议，也是一个物联网传输协议，它被设计用于轻量级的发布/订阅式的消息传输，旨在为低带宽和不稳定的网络环境中的物联网设备提供可靠的网络服务。它针对低带宽网络，低计算能力的设备，做了特殊的优化，使得其能适应各种物联网应用场景，其实现在很多的第三方推送平台也是基于 MQTT 实现的。
摘自：https://cloud.tencent.com/developer/news/223471

Centos上搭建服务：

centos安装Mosquitto

应用：

python+mqtt实现推送

其他参考：

Scrapy爬取内容去重

使用以下pipeline即可：

from scrapy.exceptions import DropItem

class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        if item['id'] in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(item['id'])
            return item

from scrapy.exceptions import DropItem

class DuplicatesPipeline(object):

def __init__(self):

self.ids_seen = set()

def process_item(self, item, spider):

if item['id'] in self.ids_seen:

raise DropItem("Duplicate item found: %s" % item)

else:

self.ids_seen.add(item['id'])

return item

【摘自：】https://docs.scrapy.org/en/latest/topics/item-pipeline.html#duplicates-filter

python 通过SSL加密方式发送邮件

def send_email(mail_user, mail_password, mail_recipient, subject, body):
	FROM = mail_user
	TO = mail_recipient if type(mail_recipient) is list else [mail_recipient]
	SUBJECT = subject
	TEXT = body

	# Prepare actual message
	message = """From: %s\nTo: %s\nSubject: %s\n\n%s
	""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
	try:
		server = smtplib.SMTP_SSL("mail.openedoo.org", 465)
		server.ehlo()
		#server.starttls()
		server.login(mail_user, mail_password)
		server.sendmail(FROM, TO, message)
		server.close()
		return 'successfully sent the mail'
	except Exception as e:
		return "failed to send mail"

def send_email(mail_user, mail_password, mail_recipient, subject, body):

FROM = mail_user

TO = mail_recipient if type(mail_recipient) is list else [mail_recipient]

SUBJECT = subject

TEXT = body

# Prepare actual message

message = """From: %s\nTo: %s\nSubject: %s\n\n%s

""" % (FROM, ", ".join(TO), SUBJECT, TEXT)

try:

server = smtplib.SMTP_SSL("mail.openedoo.org", 465)

server.ehlo()

#server.starttls()

server.login(mail_user, mail_password)

server.sendmail(FROM, TO, message)

server.close()

return 'successfully sent the mail'

except Exception as e:

return "failed to send mail"

【摘自：】https://www.programcreek.com/python/example/6443/smtplib.SMTP_SSL

webdriver的close,quit,dispose

webDriver.Close() – Close the browser window that the driver has focus of
webDriver.Quit() – Calls Dispose()
webDriver.Dispose() Closes all browser windows and safely ends the session

【摘自：】https://stackoverflow.com/questions/15067107/difference-between-webdriver-dispose-close-and-quit

Python3中如何得到Unicode码对应的中文？

先检查text是什么类型

如果type(text) is bytes，那么

text.decode('unicode_escape')

				1
2

						text.decode('unicode_escape')

如果type(text) is str，那么

text.encode('latin-1').decode('unicode_escape')

				1

						text.encode('latin-1').decode('unicode_escape')

作者：mailto1587
链接：https://www.zhihu.com/question/26921730/answer/49625649
来源：知乎

python-pip3安装pycurl

若安装时遇到报ssl相关的错误(如下)

pip3 install pycurl
Collecting pycurl
  Using cached https://files.pythonhosted.org/packages/e8/e4/0dbb8735407189f00b33d84122b9be52c790c7c3b25286826f4e1bdb7bde/pycurl-7.43.0.2.tar.gz
    Complete output from command python setup.py egg_info:
    Using curl-config (libcurl 7.54.0)
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 913, in <module>
        ext = get_extension(sys.argv, split_extension_source=split_extension_source)
      File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 582, in get_extension
        ext_config = ExtensionConfiguration(argv)
      File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 99, in __init__
        self.configure()
      File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 316, in configure_unix
        specify the SSL backend manually.''')
    __main__.ConfigurationError: Curl is configured to use SSL, but we have not been able to determine which SSL backend it is using. Please see PycURL documentation for how to specify the SSL backend manually.

    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/

pip3 install pycurl

Collecting pycurl

Using cached https://files.pythonhosted.org/packages/e8/e4/0dbb8735407189f00b33d84122b9be52c790c7c3b25286826f4e1bdb7bde/pycurl-7.43.0.2.tar.gz

Complete output from command python setup.py egg_info:

Using curl-config (libcurl 7.54.0)

Traceback (most recent call last):

File "<string>", line 1, in <module>

File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 913, in <module>

ext = get_extension(sys.argv, split_extension_source=split_extension_source)

File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 582, in get_extension

ext_config = ExtensionConfiguration(argv)

File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 99, in __init__

self.configure()

File "/private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/setup.py", line 316, in configure_unix

specify the SSL backend manually.''')

__main__.ConfigurationError: Curl is configured to use SSL, but we have not been able to determine which SSL backend it is using. Please see PycURL documentation for how to specify the SSL backend manually.

----------------------------------------

Command "python setup.py egg_info" failed with error code 1 in /private/var/folders/rt/r5cq9kls5qngd41nrn9bx6w40000gn/T/pip-install-01_xgi2x/pycurl/

解决方法参考：

新版MAC：

pip3 uninstall pycurl# 卸載庫
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS=-L/usr/local/opt/openssl/lib
export CPPFLAGS=-I/usr/local/opt/openssl/include# openssl相關頭文檔路徑
pip3 install pycurl --compile --no-cache-dir # 重新編譯安裝

pip3 uninstall pycurl# 卸載庫

export PYCURL_SSL_LIBRARY=openssl

export LDFLAGS=-L/usr/local/opt/openssl/lib

export CPPFLAGS=-I/usr/local/opt/openssl/include# openssl相關頭文檔路徑

pip3 install pycurl --compile --no-cache-dir # 重新編譯安裝

参考链接：https://hk.saowen.com/a/431b34187c78c994d278c1a1a10d480bab83b28085468aca67ce9171ea8307fc

windows上考虑安装anaconda，通过anaconda来解决：

conda install -c anaconda pycurl

1	conda install -c anaconda pycurl

参考链接：https://anaconda.org/anaconda/pycurl

Python爬虫模拟人为操作

有些大厂，像BAT可能对爬虫这类检测做的很厉害，为了模拟人为操作，需要让脚步运行的慢一些。

以下python代码用于登录支付宝账号，为了更像真人操作，

录入账号密码时，每次录入一个字符都暂停若干时间;
且暂停时间是随机时间，不固定;
同时也在一些按钮上悬停若干时间后再执行点击事件。

import random
import time

from selenium import webdriver
from selenium.webdriver import ActionChains

url = "https://auth.alipay.com/login/index.htm"
name = "你的支付宝账号"
password = "支付宝登录密码"


# 缓慢输入内容
def send_keys_delay_random(controller, keys, min_delay=0.5, max_delay=1.5):
    for key in keys:
        controller.send_keys(key)
        time.sleep(random.uniform(min_delay, max_delay))

# 延迟随机时间
def delay_time(min_delay=0.5, max_delay=1):
    time.sleep(random.uniform(min_delay, max_delay))

print("启动浏览器，打开页面")
driver = webdriver.Chrome()
driver.get(url)
delay_time()

print("选择账号密码登录")
chose_btn = driver.find_element_by_xpath('//*[@id="J-loginMethod-tabs"]/li[2]')
ActionChains(driver).move_to_element(chose_btn).click().perform()
delay_time()

print("输入账号")
username_input = driver.find_element_by_id("J-input-user")
username_input.clear()
send_keys_delay_random(username_input, name)
delay_time(1, 2)

print("输入密码")
password_input = driver.find_element_by_id("password_rsainput")
password_input.clear()
send_keys_delay_random(password_input, password)
delay_time()

print("点击登录")
btn_login = driver.find_element_by_id("J-login-btn")
ActionChains(driver).move_to_element(btn_login).click().perform()


driver.quit()
print("正常退出")

import random

import time

from selenium import webdriver

from selenium.webdriver import ActionChains

url = "https://auth.alipay.com/login/index.htm"

name = "你的支付宝账号"

password = "支付宝登录密码"

# 缓慢输入内容

def send_keys_delay_random(controller, keys, min_delay=0.5, max_delay=1.5):

for key in keys:

controller.send_keys(key)

time.sleep(random.uniform(min_delay, max_delay))

# 延迟随机时间

def delay_time(min_delay=0.5, max_delay=1):

time.sleep(random.uniform(min_delay, max_delay))

print("启动浏览器，打开页面")

driver = webdriver.Chrome()

driver.get(url)

delay_time()

print("选择账号密码登录")

chose_btn = driver.find_element_by_xpath('//*[@id="J-loginMethod-tabs"]/li[2]')

ActionChains(driver).move_to_element(chose_btn).click().perform()

delay_time()

print("输入账号")

username_input = driver.find_element_by_id("J-input-user")

username_input.clear()

send_keys_delay_random(username_input, name)

delay_time(1, 2)

print("输入密码")

password_input = driver.find_element_by_id("password_rsainput")

password_input.clear()

send_keys_delay_random(password_input, password)

delay_time()

print("点击登录")

btn_login = driver.find_element_by_id("J-login-btn")

ActionChains(driver).move_to_element(btn_login).click().perform()

driver.quit()

print("正常退出")

分类：python

jupyter文件转python文件

python中的变量&浅拷贝&深拷贝

Python中，万物皆对象！

浅拷贝

深度拷贝

mqtt

Centos上搭建服务：

应用：

Scrapy爬取内容去重

python 通过SSL加密方式发送邮件

webdriver的close,quit,dispose

Python3中如何得到Unicode码对应的中文？

python-pip3安装pycurl

Python爬虫模拟人为操作

Python中，万物皆对象！

浅拷贝

深度拷贝

Centos上搭建服务：

应用：

Cookie池：

IP代理池：